1 /*
2 * s3fs - FUSE-based file system backed by Amazon S3
3 *
4 * Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
20
21 #include <cstdio>
22 #include <cstdlib>
23 #include <unistd.h>
24 #include <dirent.h>
25 #include <pwd.h>
26 #include <sys/types.h>
27 #include <getopt.h>
28
29 #include <fstream>
30
31 #include "common.h"
32 #include "s3fs.h"
33 #include "metaheader.h"
34 #include "fdcache.h"
35 #include "fdcache_auto.h"
36 #include "curl.h"
37 #include "curl_multi.h"
38 #include "s3objlist.h"
39 #include "cache.h"
40 #include "mvnode.h"
41 #include "addhead.h"
42 #include "sighandlers.h"
43 #include "s3fs_xml.h"
44 #include "s3fs_util.h"
45 #include "string_util.h"
46 #include "s3fs_auth.h"
47 #include "s3fs_help.h"
s3fs_crypt_lib_name()48 #include "mpu_util.h"
49
50 //-------------------------------------------------------------------
51 // Symbols
52 //-------------------------------------------------------------------
53 #if !defined(ENOATTR)
54 #define ENOATTR ENODATA
55 #endif
56
57 enum dirtype {
58 DIRTYPE_UNKNOWN = -1,
59 DIRTYPE_NEW = 0,
60 DIRTYPE_OLD = 1,
61 DIRTYPE_FOLDER = 2,
62 DIRTYPE_NOOBJ = 3,
63 };
64
65 //-------------------------------------------------------------------
s3fs_destroy_global_ssl()66 // Static variables
67 //-------------------------------------------------------------------
68 static uid_t mp_uid = 0; // owner of mount point(only not specified uid opt)
69 static gid_t mp_gid = 0; // group of mount point(only not specified gid opt)
70 static mode_t mp_mode = 0; // mode of mount point
71 static mode_t mp_umask = 0; // umask for mount point
72 static bool is_mp_umask = false;// default does not set.
73 static std::string mountpoint;
74 static std::string passwd_file;
75 static std::string mimetype_file;
76 static bool nocopyapi = false;
77 static bool norenameapi = false;
78 static bool nonempty = false;
79 static bool allow_other = false;
80 static bool load_iamrole = false;
81 static uid_t s3fs_uid = 0;
82 static gid_t s3fs_gid = 0;
83 static mode_t s3fs_umask = 0;
84 static bool is_s3fs_uid = false;// default does not set.
s3fs_crypt_mutex_lock(int mode,int pos,const char * file,int line)85 static bool is_s3fs_gid = false;// default does not set.
86 static bool is_s3fs_umask = false;// default does not set.
87 static bool is_remove_cache = false;
88 static bool is_ecs = false;
89 static bool is_ibm_iam_auth = false;
90 static bool is_use_xattr = false;
91 static bool is_use_session_token = false;
92 static bool create_bucket = false;
93 static off_t multipart_threshold = 25 * 1024 * 1024;
94 static int64_t singlepart_copy_limit = 512 * 1024 * 1024;
95 static bool is_specified_endpoint = false;
96 static int s3fs_init_deferred_exit_status = 0;
97 static bool support_compat_dir = true;// default supports compatibility directory type
98 static int max_keys_list_object = 1000;// default is 1000
99 static off_t max_dirty_data = 5LL * 1024LL * 1024LL * 1024LL;
100 static bool use_wtf8 = false;
101 static off_t fake_diskfree_size = -1; // default is not set(-1)
102
103 static const char ALLBUCKET_FIELDS_TYPE[] = ""; // special key for mapping(This name is absolutely not used as a bucket name)
s3fs_crypt_get_threadid()104 static const char KEYVAL_FIELDS_TYPE[] = "\t"; // special key for mapping(This name is absolutely not used as a bucket name)
105 static const char AWS_ACCESSKEYID[] = "AWSAccessKeyId";
106 static const char AWS_SECRETKEY[] = "AWSSecretKey";
107
108 //-------------------------------------------------------------------
109 // Global functions : prototype
110 //-------------------------------------------------------------------
111 int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size = true); // [NOTE] global function because this is called from FdEntity class
s3fs_dyn_crypt_mutex(const char * file,int line)112
113 //-------------------------------------------------------------------
114 // Static functions : prototype
115 //-------------------------------------------------------------------
116 static bool is_special_name_folder_object(const char* path);
117 static int chk_dir_object_type(const char* path, std::string& newpath, std::string& nowpath, std::string& nowcache, headers_t* pmeta = NULL, dirtype* pDirType = NULL);
118 static int remove_old_type_dir(const std::string& path, dirtype type);
119 static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t* pmeta = NULL, bool overcheck = true, bool* pisforce = NULL, bool add_no_truncate_cache = false);
120 static int check_object_access(const char* path, int mask, struct stat* pstbuf);
121 static int check_object_owner(const char* path, struct stat* pstbuf);
122 static int check_parent_object_access(const char* path, int mask);
123 static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags = O_RDONLY, bool is_load = false);
124 static bool multi_head_callback(S3fsCurl* s3fscurl);
125 static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl);
126 static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler);
127 static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only = false);
128 static int directory_empty(const char* path);
s3fs_dyn_crypt_mutex_lock(int mode,struct CRYPTO_dynlock_value * dyndata,const char * file,int line)129 static int rename_large_object(const char* from, const char* to);
130 static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid);
131 static int create_directory_object(const char* path, mode_t mode, time_t atime, time_t mtime, time_t ctime, uid_t uid, gid_t gid);
132 static int rename_object(const char* from, const char* to, bool update_ctime);
133 static int rename_object_nocopy(const char* from, const char* to, bool update_ctime);
134 static int clone_directory_object(const char* from, const char* to, bool update_ctime);
135 static int rename_directory(const char* from, const char* to);
136 static int remote_mountpath_exists(const char* path);
137 static void free_xattrs(xattrs_t& xattrs);
138 static bool parse_xattr_keyval(const std::string& xattrpair, std::string& key, PXATTRVAL& pval);
139 static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
140 static std::string build_xattrs(const xattrs_t& xattrs);
141 static int s3fs_check_service();
142 static int parse_passwd_file(bucketkvmap_t& resmap);
143 static int check_for_aws_format(const kvmap_t& kvmap);
144 static int check_passwd_file_perms();
145 static int read_aws_credentials_file(const std::string &filename);
146 static int read_passwd_file();
147 static int get_access_keys();
s3fs_destroy_dyn_crypt_mutex(struct CRYPTO_dynlock_value * dyndata,const char * file,int line)148 static bool set_mountpoint_attribute(struct stat& mpst);
149 static int set_bucket(const char* arg);
150 static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_args* outargs);
151
152 //-------------------------------------------------------------------
153 // fuse interface functions
154 //-------------------------------------------------------------------
155 static int s3fs_getattr(const char* path, struct stat* stbuf);
156 static int s3fs_readlink(const char* path, char* buf, size_t size);
157 static int s3fs_mknod(const char* path, mode_t mode, dev_t rdev);
158 static int s3fs_mkdir(const char* path, mode_t mode);
159 static int s3fs_unlink(const char* path);
s3fs_init_crypt_mutex()160 static int s3fs_rmdir(const char* path);
161 static int s3fs_symlink(const char* from, const char* to);
162 static int s3fs_rename(const char* from, const char* to);
163 static int s3fs_link(const char* from, const char* to);
164 static int s3fs_chmod(const char* path, mode_t mode);
165 static int s3fs_chmod_nocopy(const char* path, mode_t mode);
166 static int s3fs_chown(const char* path, uid_t uid, gid_t gid);
167 static int s3fs_chown_nocopy(const char* path, uid_t uid, gid_t gid);
168 static int s3fs_utimens(const char* path, const struct timespec ts[2]);
169 static int s3fs_utimens_nocopy(const char* path, const struct timespec ts[2]);
170 static int s3fs_truncate(const char* path, off_t size);
171 static int s3fs_create(const char* path, mode_t mode, struct fuse_file_info* fi);
172 static int s3fs_open(const char* path, struct fuse_file_info* fi);
173 static int s3fs_read(const char* path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi);
174 static int s3fs_write(const char* path, const char* buf, size_t size, off_t offset, struct fuse_file_info* fi);
175 static int s3fs_statfs(const char* path, struct statvfs* stbuf);
176 static int s3fs_flush(const char* path, struct fuse_file_info* fi);
177 static int s3fs_fsync(const char* path, int datasync, struct fuse_file_info* fi);
178 static int s3fs_release(const char* path, struct fuse_file_info* fi);
179 static int s3fs_opendir(const char* path, struct fuse_file_info* fi);
180 static int s3fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info* fi);
181 static int s3fs_access(const char* path, int mask);
182 static void* s3fs_init(struct fuse_conn_info* conn);
183 static void s3fs_destroy(void*);
184 #if defined(__APPLE__)
185 static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags, uint32_t position);
186 static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size, uint32_t position);
187 #else
188 static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags);
189 static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size);
190 #endif
191 static int s3fs_listxattr(const char* path, char* list, size_t size);
192 static int s3fs_removexattr(const char* path, const char* name);
s3fs_destroy_crypt_mutex()193
194 //-------------------------------------------------------------------
195 // Functions
196 //-------------------------------------------------------------------
197 static bool IS_REPLACEDIR(dirtype type)
198 {
199 return DIRTYPE_OLD == type || DIRTYPE_FOLDER == type || DIRTYPE_NOOBJ == type;
200 }
201
202 static bool IS_RMTYPEDIR(dirtype type)
203 {
204 return DIRTYPE_OLD == type || DIRTYPE_FOLDER == type;
205 }
206
207 static bool is_special_name_folder_object(const char* path)
208 {
209 if(!support_compat_dir){
210 // s3fs does not support compatibility directory type("_$folder$" etc) now,
211 // thus always returns false.
212 return false;
213 }
214
215 if(!path || '\0' == path[0]){
216 return false;
217 }
218
219 std::string strpath = path;
220 headers_t header;
221
s3fs_HMAC_RAW(const void * key,size_t keylen,const unsigned char * data,size_t datalen,unsigned char ** digest,unsigned int * digestlen,bool is_sha256)222 if(std::string::npos == strpath.find("_$folder$", 0)){
223 if('/' == *strpath.rbegin()){
224 strpath.erase(strpath.length() - 1);
225 }
226 strpath += "_$folder$";
227 }
228 S3fsCurl s3fscurl;
229 if(0 != s3fscurl.HeadRequest(strpath.c_str(), header)){
230 return false;
231 }
232 header.clear();
233 S3FS_MALLOCTRIM(0);
234 return true;
235 }
236
237 // [Detail]
s3fs_HMAC(const void * key,size_t keylen,const unsigned char * data,size_t datalen,unsigned char ** digest,unsigned int * digestlen)238 // This function is complicated for checking directory object type.
239 // Arguments is used for deleting cache/path, and remake directory object.
240 // Please see the codes which calls this function.
241 //
242 // path: target path
243 // newpath: should be object path for making/putting/getting after checking
244 // nowpath: now object name for deleting after checking
245 // nowcache: now cache path for deleting after checking
246 // pmeta: headers map
247 // pDirType: directory object type
248 //
249 static int chk_dir_object_type(const char* path, std::string& newpath, std::string& nowpath, std::string& nowcache, headers_t* pmeta, dirtype* pDirType)
250 {
251 dirtype TypeTmp;
252 int result = -1;
253 bool isforce = false;
254 dirtype* pType = pDirType ? pDirType : &TypeTmp;
255
256 // Normalize new path.
257 newpath = path;
258 if('/' != *newpath.rbegin()){
259 std::string::size_type Pos;
260 if(std::string::npos != (Pos = newpath.find("_$folder$", 0))){
261 newpath.erase(Pos);
262 }
263 newpath += "/";
264 }
265
266 // Always check "dir/" at first.
267 if(0 == (result = get_object_attribute(newpath.c_str(), NULL, pmeta, false, &isforce))){
268 // Found "dir/" cache --> Check for "_$folder$", "no dir object"
269 nowcache = newpath;
270 if(is_special_name_folder_object(newpath.c_str())){ // check support_compat_dir in this function
271 // "_$folder$" type.
272 (*pType) = DIRTYPE_FOLDER;
273 nowpath.erase(newpath.length() - 1);
274 nowpath += "_$folder$"; // cut and add
275 }else if(isforce){
276 // "no dir object" type.
277 (*pType) = DIRTYPE_NOOBJ;
278 nowpath = "";
279 }else{
280 nowpath = newpath;
281 if(!nowpath.empty() && '/' == *nowpath.rbegin()){
282 // "dir/" type
283 (*pType) = DIRTYPE_NEW;
284 }else{
285 // "dir" type
286 (*pType) = DIRTYPE_OLD;
287 }
288 }
289 }else if(support_compat_dir){
290 // Check "dir" when support_compat_dir is enabled
291 nowpath.erase(newpath.length() - 1);
292 if(0 == (result = get_object_attribute(nowpath.c_str(), NULL, pmeta, false, &isforce))){
293 // Found "dir" cache --> this case is only "dir" type.
294 // Because, if object is "_$folder$" or "no dir object", the cache is "dir/" type.
295 // (But "no dir object" is checked here.)
296 nowcache = nowpath;
297 if(isforce){
298 (*pType) = DIRTYPE_NOOBJ;
299 nowpath = "";
300 }else{
301 (*pType) = DIRTYPE_OLD;
302 }
303 }else{
304 // Not found cache --> check for "_$folder$" and "no dir object".
305 // (come here is that support_compat_dir is enabled)
306 nowcache = ""; // This case is no cache.
307 nowpath += "_$folder$";
308 if(is_special_name_folder_object(nowpath.c_str())){
309 // "_$folder$" type.
310 (*pType) = DIRTYPE_FOLDER;
311 result = 0; // result is OK.
312 }else if(-ENOTEMPTY == directory_empty(newpath.c_str())){
313 // "no dir object" type.
314 (*pType) = DIRTYPE_NOOBJ;
315 nowpath = ""; // now path.
316 result = 0; // result is OK.
317 }else{
318 // Error: Unknown type.
319 (*pType) = DIRTYPE_UNKNOWN;
320 newpath = "";
321 nowpath = "";
322 }
323 }
324 }
325 return result;
326 }
327
328 static int remove_old_type_dir(const std::string& path, dirtype type)
329 {
330 if(IS_RMTYPEDIR(type)){
331 S3fsCurl s3fscurl;
332 int result = s3fscurl.DeleteRequest(path.c_str());
333 if(0 != result && -ENOENT != result){
334 return result;
335 }
336 // succeed removing or not found the directory
337 }else{
338 // nothing to do
339 }
340 return 0;
341 }
342
343 //
344 // Get object attributes with stat cache.
345 // This function is base for s3fs_getattr().
346 //
347 // [NOTICE]
348 // Checking order is changed following list because of reducing the number of the requests.
349 // 1) "dir"
350 // 2) "dir/"
351 // 3) "dir_$folder$"
352 //
353 static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t* pmeta, bool overcheck, bool* pisforce, bool add_no_truncate_cache)
354 {
355 int result = -1;
356 struct stat tmpstbuf;
357 struct stat* pstat = pstbuf ? pstbuf : &tmpstbuf;
358 headers_t tmpHead;
359 headers_t* pheader = pmeta ? pmeta : &tmpHead;
360 std::string strpath;
361 S3fsCurl s3fscurl;
362 bool forcedir = false;
363 std::string::size_type Pos;
364
365 S3FS_PRN_DBG("[path=%s]", path);
366
367 if(!path || '\0' == path[0]){
368 return -ENOENT;
369 }
370
371 memset(pstat, 0, sizeof(struct stat));
372 if(0 == strcmp(path, "/") || 0 == strcmp(path, ".")){
373 pstat->st_nlink = 1; // see fuse faq
374 pstat->st_mode = mp_mode;
375 pstat->st_uid = is_s3fs_uid ? s3fs_uid : mp_uid;
376 pstat->st_gid = is_s3fs_gid ? s3fs_gid : mp_gid;
377 return 0;
378 }
379
380 // Check cache.
381 pisforce = (NULL != pisforce ? pisforce : &forcedir);
382 (*pisforce) = false;
383 strpath = path;
384 if(support_compat_dir && overcheck && std::string::npos != (Pos = strpath.find("_$folder$", 0))){
385 strpath.erase(Pos);
386 strpath += "/";
387 }
388 if(StatCache::getStatCacheData()->GetStat(strpath, pstat, pheader, overcheck, pisforce)){
389 return 0;
390 }
391 if(StatCache::getStatCacheData()->IsNoObjectCache(strpath)){
392 // there is the path in the cache for no object, it is no object.
393 return -ENOENT;
394 }
395
396 // At first, check path
397 strpath = path;
398 result = s3fscurl.HeadRequest(strpath.c_str(), (*pheader));
399 s3fscurl.DestroyCurlHandle();
400
401 // if not found target path object, do over checking
402 if(-EPERM == result){
403 // [NOTE]
404 // In case of a permission error, it exists in directory
405 // file list but inaccessible. So there is a problem that
406 // it will send a HEAD request every time, because it is
407 // not registered in the Stats cache.
408 // Therefore, even if the file has a permission error, it
409 // should be registered in the Stats cache. However, if
410 // the response without modifying is registered in the
411 // cache, the file permission will be 0644(umask dependent)
412 // because the meta header does not exist.
413 // Thus, set the mode of 0000 here in the meta header so
414 // that s3fs can print a permission error when the file
415 // is actually accessed.
416 // It is better not to set meta header other than mode,
417 // so do not do it.
418 //
419 (*pheader)["x-amz-meta-mode"] = str(0);
420
421 }else if(0 != result){
422 if(overcheck){
423 // when support_compat_dir is disabled, strpath maybe have "_$folder$".
424 if('/' != *strpath.rbegin() && std::string::npos == strpath.find("_$folder$", 0)){
425 // now path is "object", do check "object/" for over checking
426 strpath += "/";
427 result = s3fscurl.HeadRequest(strpath.c_str(), (*pheader));
428 s3fscurl.DestroyCurlHandle();
429 }
430 if(support_compat_dir && 0 != result){
431 // now path is "object/", do check "object_$folder$" for over checking
432 strpath.erase(strpath.length() - 1);
433 strpath += "_$folder$";
434 result = s3fscurl.HeadRequest(strpath.c_str(), (*pheader));
435 s3fscurl.DestroyCurlHandle();
436
437 if(0 != result){
438 // cut "_$folder$" for over checking "no dir object" after here
439 if(std::string::npos != (Pos = strpath.find("_$folder$", 0))){
440 strpath.erase(Pos);
441 }
442 }
443 }
444 }
445 if(support_compat_dir && 0 != result && std::string::npos == strpath.find("_$folder$", 0)){
446 // now path is "object" or "object/", do check "no dir object" which is not object but has only children.
447 if('/' == *strpath.rbegin()){
448 strpath.erase(strpath.length() - 1);
449 }
450 if(-ENOTEMPTY == directory_empty(strpath.c_str())){
451 // found "no dir object".
452 strpath += "/";
453 *pisforce = true;
454 result = 0;
455 }
456 }
457 }else{
458 if(support_compat_dir && '/' != *strpath.rbegin() && std::string::npos == strpath.find("_$folder$", 0) && is_need_check_obj_detail(*pheader)){
459 // check a case of that "object" does not have attribute and "object" is possible to be directory.
460 if(-ENOTEMPTY == directory_empty(strpath.c_str())){
461 // found "no dir object".
462 strpath += "/";
463 *pisforce = true;
464 result = 0;
465 }
466 }
467 }
468
469 // [NOTE]
470 // If the file is listed but not allowed access, put it in
471 // the positive cache instead of the negative cache.
472 //
473 if(0 != result && -EPERM != result){
474 // finally, "path" object did not find. Add no object cache.
475 strpath = path; // reset original
476 StatCache::getStatCacheData()->AddNoObjectCache(strpath);
477 return result;
478 }
479
480 // if path has "_$folder$", need to cut it.
481 if(std::string::npos != (Pos = strpath.find("_$folder$", 0))){
482 strpath.erase(Pos);
483 strpath += "/";
484 }
485
486 // Set into cache
487 //
488 // [NOTE]
489 // When add_no_truncate_cache is true, the stats is always cached.
490 // This cached stats is only removed by DelStat().
491 // This is necessary for the case to access the attribute of opened file.
492 // (ex. getxattr() is called while writing to the opened file.)
493 //
494 if(add_no_truncate_cache || 0 != StatCache::getStatCacheData()->GetCacheSize()){
495 // add into stat cache
496 if(!StatCache::getStatCacheData()->AddStat(strpath, (*pheader), forcedir, add_no_truncate_cache)){
497 S3FS_PRN_ERR("failed adding stat cache [path=%s]", strpath.c_str());
498 return -ENOENT;
499 }
500 if(!StatCache::getStatCacheData()->GetStat(strpath, pstat, pheader, overcheck, pisforce)){
501 // There is not in cache.(why?) -> retry to convert.
502 if(!convert_header_to_stat(strpath.c_str(), (*pheader), pstat, forcedir)){
503 S3FS_PRN_ERR("failed convert headers to stat[path=%s]", strpath.c_str());
504 return -ENOENT;
505 }
506 }
507 }else{
508 // cache size is Zero -> only convert.
509 if(!convert_header_to_stat(strpath.c_str(), (*pheader), pstat, forcedir)){
510 S3FS_PRN_ERR("failed convert headers to stat[path=%s]", strpath.c_str());
511 return -ENOENT;
512 }
513 }
514 return 0;
515 }
516
517 //
518 // Check the object uid and gid for write/read/execute.
519 // The param "mask" is as same as access() function.
520 // If there is not a target file, this function returns -ENOENT.
521 // If the target file can be accessed, the result always is 0.
522 //
523 // path: the target object path
524 // mask: bit field(F_OK, R_OK, W_OK, X_OK) like access().
525 // stat: NULL or the pointer of struct stat.
526 //
527 static int check_object_access(const char* path, int mask, struct stat* pstbuf)
528 {
529 int result;
530 struct stat st;
531 struct stat* pst = (pstbuf ? pstbuf : &st);
532 struct fuse_context* pcxt;
533
534 S3FS_PRN_DBG("[path=%s]", path);
535
536 if(NULL == (pcxt = fuse_get_context())){
537 return -EIO;
538 }
539 S3FS_PRN_DBG("[pid=%u,uid=%u,gid=%u]", (unsigned int)(pcxt->pid), (unsigned int)(pcxt->uid), (unsigned int)(pcxt->gid));
540
541 if(0 != (result = get_object_attribute(path, pst))){
542 // If there is not the target file(object), result is -ENOENT.
543 return result;
544 }
545 if(0 == pcxt->uid){
546 // root is allowed all accessing.
547 return 0;
548 }
549 if(is_s3fs_uid && s3fs_uid == pcxt->uid){
550 // "uid" user is allowed all accessing.
551 return 0;
552 }
553 if(F_OK == mask){
554 // if there is a file, always return allowed.
555 return 0;
556 }
557
558 // for "uid", "gid" option
559 uid_t obj_uid = (is_s3fs_uid ? s3fs_uid : pst->st_uid);
560 gid_t obj_gid = (is_s3fs_gid ? s3fs_gid : pst->st_gid);
561
562 // compare file mode and uid/gid + mask.
563 mode_t mode;
564 mode_t base_mask = S_IRWXO;
565 if(is_s3fs_umask){
566 // If umask is set, all object attributes set ~umask.
567 mode = ((S_IRWXU | S_IRWXG | S_IRWXO) & ~s3fs_umask);
568 }else{
569 mode = pst->st_mode;
570 }
571 if(pcxt->uid == obj_uid){
572 base_mask |= S_IRWXU;
573 }
574 if(pcxt->gid == obj_gid){
575 base_mask |= S_IRWXG;
576 }
577 if(1 == is_uid_include_group(pcxt->uid, obj_gid)){
578 base_mask |= S_IRWXG;
579 }
580 mode &= base_mask;
581
582 if(X_OK == (mask & X_OK)){
583 if(0 == (mode & (S_IXUSR | S_IXGRP | S_IXOTH))){
584 return -EACCES;
585 }
586 }
587 if(W_OK == (mask & W_OK)){
588 if(0 == (mode & (S_IWUSR | S_IWGRP | S_IWOTH))){
589 return -EACCES;
590 }
591 }
592 if(R_OK == (mask & R_OK)){
593 if(0 == (mode & (S_IRUSR | S_IRGRP | S_IROTH))){
594 return -EACCES;
595 }
596 }
597 if(0 == mode){
598 return -EACCES;
599 }
600 return 0;
601 }
602
603 static int check_object_owner(const char* path, struct stat* pstbuf)
604 {
605 int result;
606 struct stat st;
607 struct stat* pst = (pstbuf ? pstbuf : &st);
608 struct fuse_context* pcxt;
609
610 S3FS_PRN_DBG("[path=%s]", path);
611
612 if(NULL == (pcxt = fuse_get_context())){
613 return -EIO;
614 }
615 if(0 != (result = get_object_attribute(path, pst))){
616 // If there is not the target file(object), result is -ENOENT.
617 return result;
618 }
619 // check owner
620 if(0 == pcxt->uid){
621 // root is allowed all accessing.
622 return 0;
623 }
624 if(is_s3fs_uid && s3fs_uid == pcxt->uid){
625 // "uid" user is allowed all accessing.
626 return 0;
627 }
628 if(pcxt->uid == pst->st_uid){
629 return 0;
630 }
631 return -EPERM;
632 }
633
634 //
635 // Check accessing the parent directories of the object by uid and gid.
636 //
637 static int check_parent_object_access(const char* path, int mask)
638 {
639 std::string parent;
640 int result;
641
642 S3FS_PRN_DBG("[path=%s]", path);
643
644 if(0 == strcmp(path, "/") || 0 == strcmp(path, ".")){
645 // path is mount point.
646 return 0;
647 }
648 if(X_OK == (mask & X_OK)){
649 for(parent = mydirname(path); !parent.empty(); parent = mydirname(parent)){
650 if(parent == "."){
651 parent = "/";
652 }
653 if(0 != (result = check_object_access(parent.c_str(), X_OK, NULL))){
654 return result;
655 }
656 if(parent == "/" || parent == "."){
657 break;
658 }
659 }
660 }
661 mask = (mask & ~X_OK);
662 if(0 != mask){
663 parent = mydirname(path);
664 if(parent == "."){
665 parent = "/";
666 }
667 if(0 != (result = check_object_access(parent.c_str(), mask, NULL))){
668 return result;
669 }
670 }
671 return 0;
672 }
673
674 //
675 // ssevalue is MD5 for SSE-C type, or KMS id for SSE-KMS
676 //
677 bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue)
678 {
679 if(!path){
680 return false;
681 }
682
683 headers_t meta;
684 if(0 != get_object_attribute(path, NULL, &meta)){
685 S3FS_PRN_ERR("Failed to get object(%s) headers", path);
686 return false;
687 }
688
689 ssetype = sse_type_t::SSE_DISABLE;
690 ssevalue.erase();
691 for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
692 std::string key = (*iter).first;
693 if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption") && 0 == strcasecmp((*iter).second.c_str(), "AES256")){
694 ssetype = sse_type_t::SSE_S3;
695 }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-aws-kms-key-id")){
696 ssetype = sse_type_t::SSE_KMS;
697 ssevalue = (*iter).second;
698 }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
699 ssetype = sse_type_t::SSE_C;
700 ssevalue = (*iter).second;
701 }
702 }
703 return true;
704 }
705
706 static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags, bool is_load)
707 {
708 int result;
709 struct stat stobj;
710 FdEntity* ent;
711 headers_t meta;
712
713 S3FS_PRN_INFO2("[path=%s]", path);
714
715 if(0 != (result = get_object_attribute(path, &stobj, &meta))){
716 return result;
717 }
718
719 // open
720 time_t mtime = (!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) ? -1 : stobj.st_mtime;
721 bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true;
722
723 if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, flags, force_tmpfile, true, AutoLock::NONE))){
724 S3FS_PRN_ERR("Could not open file. errno(%d)", errno);
725 return -EIO;
726 }
727 // load
728 if(is_load && !ent->LoadAll(autoent.GetPseudoFd(), &meta)){
729 S3FS_PRN_ERR("Could not load file. errno(%d)", errno);
730 autoent.Close();
731 return -EIO;
732 }
733 *entity = ent;
734 return 0;
735 }
736
737 //
738 // create or update s3 meta
739 // ow_sse_flg is for over writing sse header by use_sse option.
740 // @return fuse return code
741 //
742 int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size)
743 {
744 int result;
745 S3fsCurl s3fscurl(true);
746 off_t size;
747
748 S3FS_PRN_INFO2("[path=%s]", path);
749
750 // files larger than 5GB must be modified via the multipart interface
751 // *** If there is not target object(a case of move command),
752 // get_object_attribute() returns error with initializing buf.
753 if(use_st_size){
754 struct stat buf;
755 (void)get_object_attribute(path, &buf);
756 size = buf.st_size;
757 }else{
758 size = get_size(meta);
759 }
760
761 if(!nocopyapi && !nomultipart && size >= multipart_threshold){
762 if(0 != (result = s3fscurl.MultipartHeadRequest(path, size, meta, is_copy))){
763 return result;
764 }
765 }else{
766 if(0 != (result = s3fscurl.PutHeadRequest(path, meta, is_copy))){
767 return result;
768 }
769 }
770 return 0;
771 }
772
773 static int s3fs_getattr(const char* _path, struct stat* stbuf)
774 {
775 WTF8_ENCODE(path)
776 int result;
777
778 S3FS_PRN_INFO("[path=%s]", path);
779
780 // check parent directory attribute.
781 if(0 != (result = check_parent_object_access(path, X_OK))){
782 return result;
783 }
784 if(0 != (result = check_object_access(path, F_OK, stbuf))){
785 return result;
786 }
787 // If has already opened fd, the st_size should be instead.
788 // (See: Issue 241)
789 if(stbuf){
790 AutoFdEntity autoent;
791 FdEntity* ent;
792 if(NULL != (ent = autoent.OpenExistFdEntity(path))){
793 struct stat tmpstbuf;
794 if(ent->GetStats(tmpstbuf)){
795 stbuf->st_size = tmpstbuf.st_size;
796 }
797 }
798 stbuf->st_blksize = 4096;
799 stbuf->st_blocks = get_blocks(stbuf->st_size);
800
801 S3FS_PRN_DBG("[path=%s] uid=%u, gid=%u, mode=%04o", path, (unsigned int)(stbuf->st_uid), (unsigned int)(stbuf->st_gid), stbuf->st_mode);
802 }
803 S3FS_MALLOCTRIM(0);
804
805 return result;
806 }
807
808 static int s3fs_readlink(const char* _path, char* buf, size_t size)
809 {
810 if(!_path || !buf || 0 == size){
811 return 0;
812 }
813 WTF8_ENCODE(path)
814 std::string strValue;
815
816 // check symbolic link cache
817 if(!StatCache::getStatCacheData()->GetSymlink(std::string(path), strValue)){
818 // not found in cache, then open the path
819 { // scope for AutoFdEntity
820 AutoFdEntity autoent;
821 FdEntity* ent;
822 int result;
823 if(0 != (result = get_local_fent(autoent, &ent, path, O_RDONLY))){
824 S3FS_PRN_ERR("could not get fent(file=%s)", path);
825 return result;
826 }
827 // Get size
828 off_t readsize;
829 if(!ent->GetSize(readsize)){
830 S3FS_PRN_ERR("could not get file size(file=%s)", path);
831 return -EIO;
832 }
833 if(static_cast<off_t>(size) <= readsize){
834 readsize = size - 1;
835 }
836 // Read
837 ssize_t ressize;
838 if(0 > (ressize = ent->Read(autoent.GetPseudoFd(), buf, 0, readsize))){
839 S3FS_PRN_ERR("could not read file(file=%s, ressize=%zd)", path, ressize);
840 return static_cast<int>(ressize);
841 }
842 buf[ressize] = '\0';
843 }
844
845 // check buf if it has space words.
846 strValue = trim(std::string(buf));
847
848 // decode wtf8. This will always be shorter
849 if(use_wtf8){
850 strValue = s3fs_wtf8_decode(strValue);
851 }
852
853 // add symbolic link cache
854 if(!StatCache::getStatCacheData()->AddSymlink(std::string(path), strValue)){
855 S3FS_PRN_ERR("failed to add symbolic link cache for %s", path);
856 }
857 }
858 // copy result
859 strncpy(buf, strValue.c_str(), size - 1);
860 buf[size - 1] = '\0';
861
862 S3FS_MALLOCTRIM(0);
863
864 return 0;
865 }
866
867 static int do_create_bucket()
868 {
869 S3FS_PRN_INFO2("/");
870
871 FILE* ptmpfp;
872 int tmpfd;
873 if(endpoint == "us-east-1"){
874 ptmpfp = NULL;
875 tmpfd = -1;
876 }else{
877 if(NULL == (ptmpfp = tmpfile()) ||
878 -1 == (tmpfd = fileno(ptmpfp)) ||
879 0 >= fprintf(ptmpfp, "<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
880 " <LocationConstraint>%s</LocationConstraint>\n"
881 "</CreateBucketConfiguration>", endpoint.c_str()) ||
882 0 != fflush(ptmpfp) ||
883 -1 == fseek(ptmpfp, 0L, SEEK_SET))
884 {
885 S3FS_PRN_ERR("failed to create temporary file. err(%d)", errno);
886 if(ptmpfp){
887 fclose(ptmpfp);
888 }
889 return (0 == errno ? -EIO : -errno);
890 }
891 }
892
893 headers_t meta;
894
895 S3fsCurl s3fscurl(true);
896 int res = s3fscurl.PutRequest("/", meta, tmpfd);
897 if(res < 0){
898 long responseCode = s3fscurl.GetLastResponseCode();
899 if((responseCode == 400 || responseCode == 403) && S3fsCurl::GetSignatureType() == V2_OR_V4){
900 S3FS_PRN_ERR("Could not connect, so retry to connect by signature version 2.");
901 S3fsCurl::SetSignatureType(V2_ONLY);
902
903 // retry to check
904 s3fscurl.DestroyCurlHandle();
905 res = s3fscurl.PutRequest("/", meta, tmpfd);
906 }else if(responseCode == 409){
907 // bucket already exists
908 res = 0;
909 }
910 }
911 if(ptmpfp != NULL){
912 fclose(ptmpfp);
913 }
914 return res;
915 }
916
917 // common function for creation of a plain object
918 static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid)
919 {
920 S3FS_PRN_INFO2("[path=%s][mode=%04o]", path, mode);
921
922 time_t now = time(NULL);
923 headers_t meta;
924 meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(path));
925 meta["x-amz-meta-uid"] = str(uid);
926 meta["x-amz-meta-gid"] = str(gid);
927 meta["x-amz-meta-mode"] = str(mode);
928 meta["x-amz-meta-atime"] = str(now);
929 meta["x-amz-meta-ctime"] = str(now);
930 meta["x-amz-meta-mtime"] = str(now);
931
932 S3fsCurl s3fscurl(true);
933 return s3fscurl.PutRequest(path, meta, -1); // fd=-1 means for creating zero byte object.
934 }
935
936 static int s3fs_mknod(const char *_path, mode_t mode, dev_t rdev)
937 {
938 WTF8_ENCODE(path)
939 int result;
940 struct fuse_context* pcxt;
941
942 S3FS_PRN_INFO("[path=%s][mode=%04o][dev=%llu]", path, mode, (unsigned long long)rdev);
943
944 if(NULL == (pcxt = fuse_get_context())){
945 return -EIO;
946 }
947
948 if(0 != (result = create_file_object(path, mode, pcxt->uid, pcxt->gid))){
949 S3FS_PRN_ERR("could not create object for special file(result=%d)", result);
950 return result;
951 }
952 StatCache::getStatCacheData()->DelStat(path);
953 S3FS_MALLOCTRIM(0);
954
955 return result;
956 }
957
958 static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi)
959 {
960 WTF8_ENCODE(path)
961 int result;
962 struct fuse_context* pcxt;
963
964 S3FS_PRN_INFO("[path=%s][mode=%04o][flags=0x%x]", path, mode, fi->flags);
965
966 if(NULL == (pcxt = fuse_get_context())){
967 return -EIO;
968 }
969
970 // check parent directory attribute.
971 if(0 != (result = check_parent_object_access(path, X_OK))){
972 return result;
973 }
974 result = check_object_access(path, W_OK, NULL);
975 if(-ENOENT == result){
976 if(0 != (result = check_parent_object_access(path, W_OK))){
977 return result;
978 }
979 }else if(0 != result){
980 return result;
981 }
982
983 time_t now = time(NULL);
984 headers_t meta;
985 meta["Content-Length"] = "0";
986 meta["x-amz-meta-uid"] = str(pcxt->uid);
987 meta["x-amz-meta-gid"] = str(pcxt->gid);
988 meta["x-amz-meta-mode"] = str(mode);
989 meta["x-amz-meta-atime"] = str(now);
990 meta["x-amz-meta-mtime"] = str(now);
991 meta["x-amz-meta-ctime"] = str(now);
992
993 // [NOTE] set no_truncate flag
994 // At this point, the file has not been created(uploaded) and
995 // the data is only present in the Stats cache.
996 // The Stats cache should not be deleted automatically by
997 // timeout. If this stats is deleted, s3fs will try to get it
998 // from the server with a Head request and will get an
999 // unexpected error because the result object does not exist.
1000 //
1001 if(!StatCache::getStatCacheData()->AddStat(path, meta, false, true)){
1002 return -EIO;
1003 }
1004
1005 AutoFdEntity autoent;
1006 FdEntity* ent;
1007 if(NULL == (ent = autoent.Open(path, &meta, 0, -1, fi->flags, false, true, AutoLock::NONE))){
1008 StatCache::getStatCacheData()->DelStat(path);
1009 return -EIO;
1010 }
1011 ent->MarkDirtyNewFile();
1012 fi->fh = autoent.Detach(); // KEEP fdentity open;
1013
1014 S3FS_MALLOCTRIM(0);
1015
1016 return 0;
1017 }
1018
1019 static int create_directory_object(const char* path, mode_t mode, time_t atime, time_t mtime, time_t ctime, uid_t uid, gid_t gid)
1020 {
1021 S3FS_PRN_INFO1("[path=%s][mode=%04o][atime=%lld][mtime=%lld][ctime=%lld][uid=%u][gid=%u]", path, mode, static_cast<long long>(atime), static_cast<long long>(ctime), static_cast<long long>(mtime), (unsigned int)uid, (unsigned int)gid);
1022
1023 if(!path || '\0' == path[0]){
1024 return -EINVAL;
1025 }
1026 std::string tpath = path;
1027 if('/' != *tpath.rbegin()){
1028 tpath += "/";
1029 }
1030
1031 headers_t meta;
1032 meta["x-amz-meta-uid"] = str(uid);
1033 meta["x-amz-meta-gid"] = str(gid);
1034 meta["x-amz-meta-mode"] = str(mode);
1035 meta["x-amz-meta-atime"] = str(atime);
1036 meta["x-amz-meta-mtime"] = str(mtime);
1037 meta["x-amz-meta-ctime"] = str(ctime);
1038
1039 S3fsCurl s3fscurl;
1040 return s3fscurl.PutRequest(tpath.c_str(), meta, -1); // fd=-1 means for creating zero byte object.
1041 }
1042
1043 static int s3fs_mkdir(const char* _path, mode_t mode)
1044 {
1045 WTF8_ENCODE(path)
1046 int result;
1047 struct fuse_context* pcxt;
1048
1049 S3FS_PRN_INFO("[path=%s][mode=%04o]", path, mode);
1050
1051 if(NULL == (pcxt = fuse_get_context())){
1052 return -EIO;
1053 }
1054
1055 // check parent directory attribute.
1056 if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
1057 return result;
1058 }
1059 if(-ENOENT != (result = check_object_access(path, F_OK, NULL))){
1060 if(0 == result){
1061 result = -EEXIST;
1062 }
1063 return result;
1064 }
1065 time_t now = time(NULL);
1066 result = create_directory_object(path, mode, now, now, now, pcxt->uid, pcxt->gid);
1067
1068 StatCache::getStatCacheData()->DelStat(path);
1069 S3FS_MALLOCTRIM(0);
1070
1071 return result;
1072 }
1073
1074 static int s3fs_unlink(const char* _path)
1075 {
1076 WTF8_ENCODE(path)
1077 int result;
1078
1079 S3FS_PRN_INFO("[path=%s]", path);
1080
1081 if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
1082 return result;
1083 }
1084 S3fsCurl s3fscurl;
1085 result = s3fscurl.DeleteRequest(path);
1086 StatCache::getStatCacheData()->DelStat(path);
1087 StatCache::getStatCacheData()->DelSymlink(path);
1088 FdManager::DeleteCacheFile(path);
1089 S3FS_MALLOCTRIM(0);
1090
1091 return result;
1092 }
1093
1094 static int directory_empty(const char* path)
1095 {
1096 int result;
1097 S3ObjList head;
1098
1099 if((result = list_bucket(path, head, "/", true)) != 0){
1100 S3FS_PRN_ERR("list_bucket returns error.");
1101 return result;
1102 }
1103 if(!head.IsEmpty()){
1104 return -ENOTEMPTY;
1105 }
1106 return 0;
1107 }
1108
1109 static int s3fs_rmdir(const char* _path)
1110 {
1111 WTF8_ENCODE(path)
1112 int result;
1113 std::string strpath;
1114 struct stat stbuf;
1115
1116 S3FS_PRN_INFO("[path=%s]", path);
1117
1118 if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
1119 return result;
1120 }
1121
1122 // directory must be empty
1123 if(directory_empty(path) != 0){
1124 return -ENOTEMPTY;
1125 }
1126
1127 strpath = path;
1128 if('/' != *strpath.rbegin()){
1129 strpath += "/";
1130 }
1131 S3fsCurl s3fscurl;
1132 result = s3fscurl.DeleteRequest(strpath.c_str());
1133 s3fscurl.DestroyCurlHandle();
1134 StatCache::getStatCacheData()->DelStat(strpath.c_str());
1135
1136 // double check for old version(before 1.63)
1137 // The old version makes "dir" object, newer version makes "dir/".
1138 // A case, there is only "dir", the first removing object is "dir/".
1139 // Then "dir/" is not exists, but curl_delete returns 0.
1140 // So need to check "dir" and should be removed it.
1141 if('/' == *strpath.rbegin()){
1142 strpath.erase(strpath.length() - 1);
1143 }
1144 if(0 == get_object_attribute(strpath.c_str(), &stbuf, NULL, false)){
1145 if(S_ISDIR(stbuf.st_mode)){
1146 // Found "dir" object.
1147 result = s3fscurl.DeleteRequest(strpath.c_str());
1148 s3fscurl.DestroyCurlHandle();
1149 StatCache::getStatCacheData()->DelStat(strpath.c_str());
1150 }
1151 }
1152 // If there is no "dir" and "dir/" object(this case is made by s3cmd/s3sync),
1153 // the cache key is "dir/". So we get error only once(delete "dir/").
1154
1155 // check for "_$folder$" object.
1156 // This processing is necessary for other S3 clients compatibility.
1157 if(is_special_name_folder_object(strpath.c_str())){
1158 strpath += "_$folder$";
1159 result = s3fscurl.DeleteRequest(strpath.c_str());
1160 }
1161 S3FS_MALLOCTRIM(0);
1162
1163 return result;
1164 }
1165
1166 static int s3fs_symlink(const char* _from, const char* _to)
1167 {
1168 WTF8_ENCODE(from)
1169 WTF8_ENCODE(to)
1170 int result;
1171 struct fuse_context* pcxt;
1172
1173 S3FS_PRN_INFO("[from=%s][to=%s]", from, to);
1174
1175 if(NULL == (pcxt = fuse_get_context())){
1176 return -EIO;
1177 }
1178 if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
1179 return result;
1180 }
1181 if(-ENOENT != (result = check_object_access(to, F_OK, NULL))){
1182 if(0 == result){
1183 result = -EEXIST;
1184 }
1185 return result;
1186 }
1187
1188 time_t now = time(NULL);
1189 headers_t headers;
1190 headers["Content-Type"] = std::string("application/octet-stream"); // Static
1191 headers["x-amz-meta-mode"] = str(S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO);
1192 headers["x-amz-meta-atime"] = str(now);
1193 headers["x-amz-meta-ctime"] = str(now);
1194 headers["x-amz-meta-mtime"] = str(now);
1195 headers["x-amz-meta-uid"] = str(pcxt->uid);
1196 headers["x-amz-meta-gid"] = str(pcxt->gid);
1197
1198 // open tmpfile
1199 std::string strFrom;
1200 { // scope for AutoFdEntity
1201 AutoFdEntity autoent;
1202 FdEntity* ent;
1203 if(NULL == (ent = autoent.Open(to, &headers, 0, -1, O_RDWR, true, true, AutoLock::NONE))){
1204 S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno);
1205 return -errno;
1206 }
1207 // write(without space words)
1208 strFrom = trim(std::string(from));
1209 ssize_t from_size = static_cast<ssize_t>(strFrom.length());
1210 ssize_t ressize;
1211 if(from_size != (ressize = ent->Write(autoent.GetPseudoFd(), strFrom.c_str(), 0, from_size))){
1212 if(ressize < 0){
1213 S3FS_PRN_ERR("could not write tmpfile(errno=%d)", static_cast<int>(ressize));
1214 return static_cast<int>(ressize);
1215 }else{
1216 S3FS_PRN_ERR("could not write tmpfile %zd byte(errno=%d)", ressize, errno);
1217 return (0 == errno ? -EIO : -errno);
1218 }
1219 }
1220 // upload
1221 if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
1222 S3FS_PRN_WARN("could not upload tmpfile(result=%d)", result);
1223 }
1224 }
1225
1226 StatCache::getStatCacheData()->DelStat(to);
1227 if(!StatCache::getStatCacheData()->AddSymlink(std::string(to), strFrom)){
1228 S3FS_PRN_ERR("failed to add symbolic link cache for %s", to);
1229 }
1230 S3FS_MALLOCTRIM(0);
1231
1232 return result;
1233 }
1234
1235 static int rename_object(const char* from, const char* to, bool update_ctime)
1236 {
1237 int result;
1238 std::string s3_realpath;
1239 headers_t meta;
1240 struct stat buf;
1241
1242 S3FS_PRN_INFO1("[from=%s][to=%s]", from , to);
1243
1244 if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
1245 // not permit writing "to" object parent dir.
1246 return result;
1247 }
1248 if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
1249 // not permit removing "from" object parent dir.
1250 return result;
1251 }
1252 if(0 != (result = get_object_attribute(from, &buf, &meta))){
1253 return result;
1254 }
1255 s3_realpath = get_realpath(from);
1256
1257 if(update_ctime){
1258 meta["x-amz-meta-ctime"] = str(time(NULL));
1259 }
1260 meta["x-amz-copy-source"] = urlEncode(service_path + bucket + s3_realpath);
1261 meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(to));
1262 meta["x-amz-metadata-directive"] = "REPLACE";
1263
1264 // [NOTE]
1265 // If it has a cache, open it first and leave it open until rename.
1266 // The cache is renamed after put_header, because it must be open
1267 // at the time of renaming.
1268 {
1269 // update time
1270 AutoFdEntity autoent;
1271 FdEntity* ent;
1272 if(NULL == (ent = autoent.OpenExistFdEntity(from))){
1273 // no opened fd
1274 if(FdManager::IsCacheDir()){
1275 // create cache file if be needed
1276 ent = autoent.Open(from, &meta, buf.st_size, -1, O_RDONLY, false, true, AutoLock::NONE);
1277 }
1278 if(ent){
1279 struct timespec mtime = get_mtime(meta);
1280 struct timespec ctime = get_ctime(meta);
1281 struct timespec atime = get_atime(meta);
1282 if(mtime.tv_sec < 0){
1283 mtime.tv_sec = 0L;
1284 mtime.tv_nsec = 0L;
1285 }
1286 if(ctime.tv_sec < 0){
1287 ctime.tv_sec = 0L;
1288 ctime.tv_nsec = 0L;
1289 }
1290 if(atime.tv_sec < 0){
1291 atime.tv_sec = 0L;
1292 atime.tv_nsec = 0L;
1293 }
1294 ent->SetMCtime(mtime, ctime);
1295 ent->SetAtime(atime);
1296 }
1297 }
1298
1299 // copy
1300 if(0 != (result = put_headers(to, meta, true, /* use_st_size= */ false))){
1301 return result;
1302 }
1303
1304 // rename
1305 FdManager::get()->Rename(from, to);
1306 }
1307
1308 // Remove file
1309 result = s3fs_unlink(from);
1310
1311 StatCache::getStatCacheData()->DelStat(to);
1312
1313 return result;
1314 }
1315
1316 static int rename_object_nocopy(const char* from, const char* to, bool update_ctime)
1317 {
1318 int result;
1319
1320 S3FS_PRN_INFO1("[from=%s][to=%s]", from , to);
1321
1322 if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
1323 // not permit writing "to" object parent dir.
1324 return result;
1325 }
1326 if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
1327 // not permit removing "from" object parent dir.
1328 return result;
1329 }
1330
1331 // open & load
1332 { // scope for AutoFdEntity
1333 AutoFdEntity autoent;
1334 FdEntity* ent;
1335 if(0 != (result = get_local_fent(autoent, &ent, from, O_RDWR, true))){
1336 S3FS_PRN_ERR("could not open and read file(%s)", from);
1337 return result;
1338 }
1339
1340 // Set header
1341 if(!ent->SetContentType(to)){
1342 S3FS_PRN_ERR("could not set content-type for %s", to);
1343 return -EIO;
1344 }
1345
1346 // update ctime
1347 if(update_ctime){
1348 struct timespec ts = {time(NULL), 0};
1349 ent->SetCtime(ts);
1350 }
1351
1352 // upload
1353 if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), to, true))){
1354 S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result);
1355 return result;
1356 }
1357 FdManager::get()->Rename(from, to);
1358 }
1359
1360 // Remove file
1361 result = s3fs_unlink(from);
1362
1363 // Stats
1364 StatCache::getStatCacheData()->DelStat(to);
1365
1366 return result;
1367 }
1368
1369 static int rename_large_object(const char* from, const char* to)
1370 {
1371 int result;
1372 struct stat buf;
1373 headers_t meta;
1374
1375 S3FS_PRN_INFO1("[from=%s][to=%s]", from , to);
1376
1377 if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
1378 // not permit writing "to" object parent dir.
1379 return result;
1380 }
1381 if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
1382 // not permit removing "from" object parent dir.
1383 return result;
1384 }
1385 if(0 != (result = get_object_attribute(from, &buf, &meta, false))){
1386 return result;
1387 }
1388
1389 S3fsCurl s3fscurl(true);
1390 if(0 != (result = s3fscurl.MultipartRenameRequest(from, to, meta, buf.st_size))){
1391 return result;
1392 }
1393 s3fscurl.DestroyCurlHandle();
1394
1395 // Remove file
1396 result = s3fs_unlink(from);
1397
1398 StatCache::getStatCacheData()->DelStat(to);
1399 FdManager::DeleteCacheFile(to);
1400
1401 return result;
1402 }
1403
1404 static int clone_directory_object(const char* from, const char* to, bool update_ctime)
1405 {
1406 int result = -1;
1407 struct stat stbuf;
1408
1409 S3FS_PRN_INFO1("[from=%s][to=%s]", from, to);
1410
1411 // get target's attributes
1412 if(0 != (result = get_object_attribute(from, &stbuf))){
1413 return result;
1414 }
1415 result = create_directory_object(to, stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, (update_ctime ? time(NULL) : stbuf.st_ctime), stbuf.st_uid, stbuf.st_gid);
1416
1417 StatCache::getStatCacheData()->DelStat(to);
1418
1419 return result;
1420 }
1421
1422 static int rename_directory(const char* from, const char* to)
1423 {
1424 S3ObjList head;
1425 s3obj_list_t headlist;
1426 std::string strfrom = from ? from : ""; // from is without "/".
1427 std::string strto = to ? to : ""; // to is without "/" too.
1428 std::string basepath = strfrom + "/";
1429 std::string newpath; // should be from name(not used)
1430 std::string nowcache; // now cache path(not used)
1431 dirtype DirType;
1432 bool normdir;
1433 MVNODE* mn_head = NULL;
1434 MVNODE* mn_tail = NULL;
1435 MVNODE* mn_cur;
1436 struct stat stbuf;
1437 int result;
1438 bool is_dir;
1439
1440 S3FS_PRN_INFO1("[from=%s][to=%s]", from, to);
1441
1442 //
1443 // Initiate and Add base directory into MVNODE struct.
1444 //
1445 strto += "/";
1446 if(0 == chk_dir_object_type(from, newpath, strfrom, nowcache, NULL, &DirType) && DIRTYPE_UNKNOWN != DirType){
1447 if(DIRTYPE_NOOBJ != DirType){
1448 normdir = false;
1449 }else{
1450 normdir = true;
1451 strfrom = from; // from directory is not removed, but from directory attr is needed.
1452 }
1453 if(NULL == (add_mvnode(&mn_head, &mn_tail, strfrom.c_str(), strto.c_str(), true, normdir))){
1454 return -ENOMEM;
1455 }
1456 }else{
1457 // Something wrong about "from" directory.
1458 }
1459
1460 //
1461 // get a list of all the objects
1462 //
1463 // No delimiter is specified, the result(head) is all object keys.
1464 // (CommonPrefixes is empty, but all object is listed in Key.)
1465 if(0 != (result = list_bucket(basepath.c_str(), head, NULL))){
1466 S3FS_PRN_ERR("list_bucket returns error.");
1467 return result;
1468 }
1469 head.GetNameList(headlist); // get name without "/".
1470 S3ObjList::MakeHierarchizedList(headlist, false); // add hierarchized dir.
1471
1472 s3obj_list_t::const_iterator liter;
1473 for(liter = headlist.begin(); headlist.end() != liter; ++liter){
1474 // make "from" and "to" object name.
1475 std::string from_name = basepath + (*liter);
1476 std::string to_name = strto + (*liter);
1477 std::string etag = head.GetETag((*liter).c_str());
1478
1479 // Check subdirectory.
1480 StatCache::getStatCacheData()->HasStat(from_name, etag.c_str()); // Check ETag
1481 if(0 != get_object_attribute(from_name.c_str(), &stbuf, NULL)){
1482 S3FS_PRN_WARN("failed to get %s object attribute.", from_name.c_str());
1483 continue;
1484 }
1485 if(S_ISDIR(stbuf.st_mode)){
1486 is_dir = true;
1487 if(0 != chk_dir_object_type(from_name.c_str(), newpath, from_name, nowcache, NULL, &DirType) || DIRTYPE_UNKNOWN == DirType){
1488 S3FS_PRN_WARN("failed to get %s%s object directory type.", basepath.c_str(), (*liter).c_str());
1489 continue;
1490 }
1491 if(DIRTYPE_NOOBJ != DirType){
1492 normdir = false;
1493 }else{
1494 normdir = true;
1495 from_name = basepath + (*liter); // from directory is not removed, but from directory attr is needed.
1496 }
1497 }else{
1498 is_dir = false;
1499 normdir = false;
1500 }
1501
1502 // push this one onto the stack
1503 if(NULL == add_mvnode(&mn_head, &mn_tail, from_name.c_str(), to_name.c_str(), is_dir, normdir)){
1504 return -ENOMEM;
1505 }
1506 }
1507
1508 //
1509 // rename
1510 //
1511 // rename directory objects.
1512 for(mn_cur = mn_head; mn_cur; mn_cur = mn_cur->next){
1513 if(mn_cur->is_dir && mn_cur->old_path && '\0' != mn_cur->old_path[0]){
1514 // [NOTE]
1515 // The ctime is updated only for the top (from) directory.
1516 // Other than that, it will not be updated.
1517 //
1518 if(0 != (result = clone_directory_object(mn_cur->old_path, mn_cur->new_path, (strfrom == mn_cur->old_path)))){
1519 S3FS_PRN_ERR("clone_directory_object returned an error(%d)", result);
1520 free_mvnodes(mn_head);
1521 return result;
1522 }
1523 }
1524 }
1525
1526 // iterate over the list - copy the files with rename_object
1527 // does a safe copy - copies first and then deletes old
1528 for(mn_cur = mn_head; mn_cur; mn_cur = mn_cur->next){
1529 if(!mn_cur->is_dir){
1530 if(!nocopyapi && !norenameapi){
1531 result = rename_object(mn_cur->old_path, mn_cur->new_path, false); // keep ctime
1532 }else{
1533 result = rename_object_nocopy(mn_cur->old_path, mn_cur->new_path, false); // keep ctime
1534 }
1535 if(0 != result){
1536 S3FS_PRN_ERR("rename_object returned an error(%d)", result);
1537 free_mvnodes(mn_head);
1538 return result;
1539 }
1540 }
1541 }
1542
1543 // Iterate over old the directories, bottoms up and remove
1544 for(mn_cur = mn_tail; mn_cur; mn_cur = mn_cur->prev){
1545 if(mn_cur->is_dir && mn_cur->old_path && '\0' != mn_cur->old_path[0]){
1546 if(!(mn_cur->is_normdir)){
1547 if(0 != (result = s3fs_rmdir(mn_cur->old_path))){
1548 S3FS_PRN_ERR("s3fs_rmdir returned an error(%d)", result);
1549 free_mvnodes(mn_head);
1550 return result;
1551 }
1552 }else{
1553 // cache clear.
1554 StatCache::getStatCacheData()->DelStat(mn_cur->old_path);
1555 }
1556 }
1557 }
1558 free_mvnodes(mn_head);
1559
1560 return 0;
1561 }
1562
1563 static int s3fs_rename(const char* _from, const char* _to)
1564 {
1565 WTF8_ENCODE(from)
1566 WTF8_ENCODE(to)
1567 struct stat buf;
1568 int result;
1569
1570 S3FS_PRN_INFO("[from=%s][to=%s]", from, to);
1571
1572 if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
1573 // not permit writing "to" object parent dir.
1574 return result;
1575 }
1576 if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
1577 // not permit removing "from" object parent dir.
1578 return result;
1579 }
1580 if(0 != (result = get_object_attribute(from, &buf, NULL))){
1581 return result;
1582 }
1583 if(0 != (result = directory_empty(to))){
1584 return result;
1585 }
1586
1587 // flush pending writes if file is open
1588 { // scope for AutoFdEntity
1589 AutoFdEntity autoent;
1590 FdEntity* ent;
1591 if(NULL != (ent = autoent.OpenExistFdEntity(from, O_RDWR))){
1592 if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
1593 S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result);
1594 return result;
1595 }
1596 StatCache::getStatCacheData()->DelStat(from);
1597 }
1598 }
1599
1600 // files larger than 5GB must be modified via the multipart interface
1601 if(S_ISDIR(buf.st_mode)){
1602 result = rename_directory(from, to);
1603 }else if(!nomultipart && buf.st_size >= singlepart_copy_limit){
1604 result = rename_large_object(from, to);
1605 }else{
1606 if(!nocopyapi && !norenameapi){
1607 result = rename_object(from, to, true); // update ctime
1608 }else{
1609 result = rename_object_nocopy(from, to, true); // update ctime
1610 }
1611 }
1612 S3FS_MALLOCTRIM(0);
1613
1614 return result;
1615 }
1616
1617 static int s3fs_link(const char* _from, const char* _to)
1618 {
1619 WTF8_ENCODE(from)
1620 WTF8_ENCODE(to)
1621 S3FS_PRN_INFO("[from=%s][to=%s]", from, to);
1622 return -ENOTSUP;
1623 }
1624
1625 static int s3fs_chmod(const char* _path, mode_t mode)
1626 {
1627 WTF8_ENCODE(path)
1628 int result;
1629 std::string strpath;
1630 std::string newpath;
1631 std::string nowcache;
1632 headers_t meta;
1633 struct stat stbuf;
1634 dirtype nDirType = DIRTYPE_UNKNOWN;
1635
1636 S3FS_PRN_INFO("[path=%s][mode=%04o]", path, mode);
1637
1638 if(0 == strcmp(path, "/")){
1639 S3FS_PRN_ERR("Could not change mode for mount point.");
1640 return -EIO;
1641 }
1642 if(0 != (result = check_parent_object_access(path, X_OK))){
1643 return result;
1644 }
1645 if(0 != (result = check_object_owner(path, &stbuf))){
1646 return result;
1647 }
1648
1649 if(S_ISDIR(stbuf.st_mode)){
1650 result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
1651 }else{
1652 strpath = path;
1653 nowcache = strpath;
1654 result = get_object_attribute(strpath.c_str(), NULL, &meta);
1655 }
1656 if(0 != result){
1657 return result;
1658 }
1659
1660 if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
1661 // Should rebuild directory object(except new type)
1662 // Need to remove old dir("dir" etc) and make new dir("dir/")
1663
1664 // At first, remove directory old object
1665 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
1666 return result;
1667 }
1668 StatCache::getStatCacheData()->DelStat(nowcache);
1669
1670 // Make new directory object("dir/")
1671 if(0 != (result = create_directory_object(newpath.c_str(), mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), stbuf.st_uid, stbuf.st_gid))){
1672 return result;
1673 }
1674 }else{
1675 // normal object or directory object of newer version
1676 headers_t updatemeta;
1677 updatemeta["x-amz-meta-ctime"] = str(time(NULL));
1678 updatemeta["x-amz-meta-mode"] = str(mode);
1679 updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
1680 updatemeta["x-amz-metadata-directive"] = "REPLACE";
1681
1682 // check opened file handle.
1683 //
1684 // If the file starts uploading by multipart when the disk capacity is insufficient,
1685 // we need to put these header after finishing upload.
1686 // Or if the file is only open, we must update to FdEntity's internal meta.
1687 //
1688 AutoFdEntity autoent;
1689 FdEntity* ent;
1690 bool need_put_header = true;
1691 if(NULL != (ent = autoent.OpenExistFdEntity(path))){
1692 if(ent->MergeOrgMeta(updatemeta)){
1693 // meta is changed, but now uploading.
1694 // then the meta is pending and accumulated to be put after the upload is complete.
1695 S3FS_PRN_INFO("meta pending until upload is complete");
1696 need_put_header = false;
1697
1698 // If there is data in the Stats cache, update the Stats cache.
1699 StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta);
1700 }
1701 }
1702 if(need_put_header){
1703 // not found opened file.
1704 merge_headers(meta, updatemeta, true);
1705
1706 // upload meta directly.
1707 if(0 != (result = put_headers(strpath.c_str(), meta, true))){
1708 return result;
1709 }
1710 StatCache::getStatCacheData()->DelStat(nowcache);
1711 }
1712 }
1713 S3FS_MALLOCTRIM(0);
1714
1715 return 0;
1716 }
1717
1718 static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
1719 {
1720 WTF8_ENCODE(path)
1721 int result;
1722 std::string strpath;
1723 std::string newpath;
1724 std::string nowcache;
1725 struct stat stbuf;
1726 dirtype nDirType = DIRTYPE_UNKNOWN;
1727
1728 S3FS_PRN_INFO1("[path=%s][mode=%04o]", path, mode);
1729
1730 if(0 == strcmp(path, "/")){
1731 S3FS_PRN_ERR("Could not change mode for mount point.");
1732 return -EIO;
1733 }
1734 if(0 != (result = check_parent_object_access(path, X_OK))){
1735 return result;
1736 }
1737 if(0 != (result = check_object_owner(path, &stbuf))){
1738 return result;
1739 }
1740
1741 // Get attributes
1742 if(S_ISDIR(stbuf.st_mode)){
1743 result = chk_dir_object_type(path, newpath, strpath, nowcache, NULL, &nDirType);
1744 }else{
1745 strpath = path;
1746 nowcache = strpath;
1747 result = get_object_attribute(strpath.c_str(), NULL, NULL);
1748 }
1749 if(0 != result){
1750 return result;
1751 }
1752
1753 if(S_ISDIR(stbuf.st_mode)){
1754 // Should rebuild all directory object
1755 // Need to remove old dir("dir" etc) and make new dir("dir/")
1756
1757 // At first, remove directory old object
1758 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
1759 return result;
1760 }
1761 StatCache::getStatCacheData()->DelStat(nowcache);
1762
1763 // Make new directory object("dir/")
1764 if(0 != (result = create_directory_object(newpath.c_str(), mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), stbuf.st_uid, stbuf.st_gid))){
1765 return result;
1766 }
1767 }else{
1768 // normal object or directory object of newer version
1769
1770 // open & load
1771 AutoFdEntity autoent;
1772 FdEntity* ent;
1773 if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
1774 S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
1775 return result;
1776 }
1777
1778 struct timespec ts = {time(NULL), 0};
1779 ent->SetCtime(ts);
1780
1781 // Change file mode
1782 ent->SetMode(mode);
1783
1784 // upload
1785 if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
1786 S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
1787 return result;
1788 }
1789 StatCache::getStatCacheData()->DelStat(nowcache);
1790 }
1791 S3FS_MALLOCTRIM(0);
1792
1793 return result;
1794 }
1795
1796 static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
1797 {
1798 WTF8_ENCODE(path)
1799 int result;
1800 std::string strpath;
1801 std::string newpath;
1802 std::string nowcache;
1803 headers_t meta;
1804 struct stat stbuf;
1805 dirtype nDirType = DIRTYPE_UNKNOWN;
1806
1807 S3FS_PRN_INFO("[path=%s][uid=%u][gid=%u]", path, (unsigned int)uid, (unsigned int)gid);
1808
1809 if(0 == strcmp(path, "/")){
1810 S3FS_PRN_ERR("Could not change owner for mount point.");
1811 return -EIO;
1812 }
1813 if(0 != (result = check_parent_object_access(path, X_OK))){
1814 return result;
1815 }
1816 if(0 != (result = check_object_owner(path, &stbuf))){
1817 return result;
1818 }
1819
1820 if((uid_t)(-1) == uid){
1821 uid = stbuf.st_uid;
1822 }
1823 if((gid_t)(-1) == gid){
1824 gid = stbuf.st_gid;
1825 }
1826 if(S_ISDIR(stbuf.st_mode)){
1827 result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
1828 }else{
1829 strpath = path;
1830 nowcache = strpath;
1831 result = get_object_attribute(strpath.c_str(), NULL, &meta);
1832 }
1833 if(0 != result){
1834 return result;
1835 }
1836
1837 if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
1838 // Should rebuild directory object(except new type)
1839 // Need to remove old dir("dir" etc) and make new dir("dir/")
1840
1841 // At first, remove directory old object
1842 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
1843 return result;
1844 }
1845 StatCache::getStatCacheData()->DelStat(nowcache);
1846
1847 // Make new directory object("dir/")
1848 if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), uid, gid))){
1849 return result;
1850 }
1851 }else{
1852 headers_t updatemeta;
1853 updatemeta["x-amz-meta-ctime"] = str(time(NULL));
1854 updatemeta["x-amz-meta-uid"] = str(uid);
1855 updatemeta["x-amz-meta-gid"] = str(gid);
1856 updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
1857 updatemeta["x-amz-metadata-directive"] = "REPLACE";
1858
1859 // check opened file handle.
1860 //
1861 // If the file starts uploading by multipart when the disk capacity is insufficient,
1862 // we need to put these header after finishing upload.
1863 // Or if the file is only open, we must update to FdEntity's internal meta.
1864 //
1865 AutoFdEntity autoent;
1866 FdEntity* ent;
1867 bool need_put_header = true;
1868 if(NULL != (ent = autoent.OpenExistFdEntity(path))){
1869 if(ent->MergeOrgMeta(updatemeta)){
1870 // meta is changed, but now uploading.
1871 // then the meta is pending and accumulated to be put after the upload is complete.
1872 S3FS_PRN_INFO("meta pending until upload is complete");
1873 need_put_header = false;
1874
1875 // If there is data in the Stats cache, update the Stats cache.
1876 StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta);
1877 }
1878 }
1879 if(need_put_header){
1880 // not found opened file.
1881 merge_headers(meta, updatemeta, true);
1882
1883 // upload meta directly.
1884 if(0 != (result = put_headers(strpath.c_str(), meta, true))){
1885 return result;
1886 }
1887 StatCache::getStatCacheData()->DelStat(nowcache);
1888 }
1889 }
1890 S3FS_MALLOCTRIM(0);
1891
1892 return 0;
1893 }
1894
1895 static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
1896 {
1897 WTF8_ENCODE(path)
1898 int result;
1899 std::string strpath;
1900 std::string newpath;
1901 std::string nowcache;
1902 struct stat stbuf;
1903 dirtype nDirType = DIRTYPE_UNKNOWN;
1904
1905 S3FS_PRN_INFO1("[path=%s][uid=%u][gid=%u]", path, (unsigned int)uid, (unsigned int)gid);
1906
1907 if(0 == strcmp(path, "/")){
1908 S3FS_PRN_ERR("Could not change owner for mount point.");
1909 return -EIO;
1910 }
1911 if(0 != (result = check_parent_object_access(path, X_OK))){
1912 return result;
1913 }
1914 if(0 != (result = check_object_owner(path, &stbuf))){
1915 return result;
1916 }
1917
1918 if((uid_t)(-1) == uid){
1919 uid = stbuf.st_uid;
1920 }
1921 if((gid_t)(-1) == gid){
1922 gid = stbuf.st_gid;
1923 }
1924
1925 // Get attributes
1926 if(S_ISDIR(stbuf.st_mode)){
1927 result = chk_dir_object_type(path, newpath, strpath, nowcache, NULL, &nDirType);
1928 }else{
1929 strpath = path;
1930 nowcache = strpath;
1931 result = get_object_attribute(strpath.c_str(), NULL, NULL);
1932 }
1933 if(0 != result){
1934 return result;
1935 }
1936
1937 if(S_ISDIR(stbuf.st_mode)){
1938 // Should rebuild all directory object
1939 // Need to remove old dir("dir" etc) and make new dir("dir/")
1940
1941 // At first, remove directory old object
1942 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
1943 return result;
1944 }
1945 StatCache::getStatCacheData()->DelStat(nowcache);
1946
1947 // Make new directory object("dir/")
1948 if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), uid, gid))){
1949 return result;
1950 }
1951 }else{
1952 // normal object or directory object of newer version
1953
1954 // open & load
1955 AutoFdEntity autoent;
1956 FdEntity* ent;
1957 if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
1958 S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
1959 return result;
1960 }
1961
1962 struct timespec ts = {time(NULL), 0};
1963 ent->SetCtime(ts);
1964
1965 // Change owner
1966 ent->SetUId(uid);
1967 ent->SetGId(gid);
1968
1969 // upload
1970 if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
1971 S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
1972 return result;
1973 }
1974 StatCache::getStatCacheData()->DelStat(nowcache);
1975 }
1976 S3FS_MALLOCTRIM(0);
1977
1978 return result;
1979 }
1980
1981 static int s3fs_utimens(const char* _path, const struct timespec ts[2])
1982 {
1983 WTF8_ENCODE(path)
1984 int result;
1985 std::string strpath;
1986 std::string newpath;
1987 std::string nowcache;
1988 headers_t meta;
1989 struct stat stbuf;
1990 dirtype nDirType = DIRTYPE_UNKNOWN;
1991
1992 S3FS_PRN_INFO("[path=%s][mtime=%lld][ctime/atime=%lld]", path, static_cast<long long>(ts[1].tv_sec), static_cast<long long>(ts[0].tv_sec));
1993
1994 if(0 == strcmp(path, "/")){
1995 S3FS_PRN_ERR("Could not change mtime for mount point.");
1996 return -EIO;
1997 }
1998 if(0 != (result = check_parent_object_access(path, X_OK))){
1999 return result;
2000 }
2001 if(0 != (result = check_object_access(path, W_OK, &stbuf))){
2002 if(0 != check_object_owner(path, &stbuf)){
2003 return result;
2004 }
2005 }
2006
2007 if(S_ISDIR(stbuf.st_mode)){
2008 result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
2009 }else{
2010 strpath = path;
2011 nowcache = strpath;
2012 result = get_object_attribute(strpath.c_str(), NULL, &meta);
2013 }
2014 if(0 != result){
2015 return result;
2016 }
2017
2018 if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
2019 // Should rebuild directory object(except new type)
2020 // Need to remove old dir("dir" etc) and make new dir("dir/")
2021
2022 // At first, remove directory old object
2023 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
2024 return result;
2025 }
2026 StatCache::getStatCacheData()->DelStat(nowcache);
2027
2028 // Make new directory object("dir/")
2029 if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts[0].tv_sec, ts[1].tv_sec, ts[0].tv_sec, stbuf.st_uid, stbuf.st_gid))){
2030 return result;
2031 }
2032 }else{
2033 headers_t updatemeta;
2034 updatemeta["x-amz-meta-mtime"] = str(ts[1]);
2035 updatemeta["x-amz-meta-ctime"] = str(ts[0]);
2036 updatemeta["x-amz-meta-atime"] = str(ts[0]);
2037 updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
2038 updatemeta["x-amz-metadata-directive"] = "REPLACE";
2039
2040 // check opened file handle.
2041 //
2042 // If the file starts uploading by multipart when the disk capacity is insufficient,
2043 // we need to put these header after finishing upload.
2044 // Or if the file is only open, we must update to FdEntity's internal meta.
2045 //
2046 AutoFdEntity autoent;
2047 FdEntity* ent;
2048 bool need_put_header = true;
2049 bool keep_mtime = false;
2050 if(NULL != (ent = autoent.OpenExistFdEntity(path))){
2051 if(ent->MergeOrgMeta(updatemeta)){
2052 // meta is changed, but now uploading.
2053 // then the meta is pending and accumulated to be put after the upload is complete.
2054 S3FS_PRN_INFO("meta pending until upload is complete");
2055 need_put_header = false;
2056 ent->SetHoldingMtime(ts[1]); // ts[1] is mtime
2057
2058 // If there is data in the Stats cache, update the Stats cache.
2059 StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta);
2060
2061 }else{
2062 S3FS_PRN_INFO("meta is not pending, but need to keep current mtime.");
2063
2064 // [NOTE]
2065 // Depending on the order in which write/flush and utimens are called,
2066 // the mtime updated here may be overwritten at the time of flush.
2067 // To avoid that, set a special flag.
2068 //
2069 keep_mtime = true;
2070 }
2071 }
2072 if(need_put_header){
2073 // not found opened file.
2074 merge_headers(meta, updatemeta, true);
2075
2076 // upload meta directly.
2077 if(0 != (result = put_headers(strpath.c_str(), meta, true))){
2078 return result;
2079 }
2080 StatCache::getStatCacheData()->DelStat(nowcache);
2081
2082 if(keep_mtime){
2083 ent->SetHoldingMtime(ts[1]); // ts[1].tv_sec is mtime
2084 }
2085 }
2086 }
2087 S3FS_MALLOCTRIM(0);
2088
2089 return 0;
2090 }
2091
2092 static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
2093 {
2094 WTF8_ENCODE(path)
2095 int result;
2096 std::string strpath;
2097 std::string newpath;
2098 std::string nowcache;
2099 struct stat stbuf;
2100 dirtype nDirType = DIRTYPE_UNKNOWN;
2101
2102 S3FS_PRN_INFO1("[path=%s][mtime=%lld][atime/ctime=%lld]", path, static_cast<long long>(ts[1].tv_sec), static_cast<long long>(ts[0].tv_sec));
2103
2104 if(0 == strcmp(path, "/")){
2105 S3FS_PRN_ERR("Could not change mtime for mount point.");
2106 return -EIO;
2107 }
2108 if(0 != (result = check_parent_object_access(path, X_OK))){
2109 return result;
2110 }
2111 if(0 != (result = check_object_access(path, W_OK, &stbuf))){
2112 if(0 != check_object_owner(path, &stbuf)){
2113 return result;
2114 }
2115 }
2116
2117 // Get attributes
2118 if(S_ISDIR(stbuf.st_mode)){
2119 result = chk_dir_object_type(path, newpath, strpath, nowcache, NULL, &nDirType);
2120 }else{
2121 strpath = path;
2122 nowcache = strpath;
2123 result = get_object_attribute(strpath.c_str(), NULL, NULL);
2124 }
2125 if(0 != result){
2126 return result;
2127 }
2128
2129 if(S_ISDIR(stbuf.st_mode)){
2130 // Should rebuild all directory object
2131 // Need to remove old dir("dir" etc) and make new dir("dir/")
2132
2133 // At first, remove directory old object
2134 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
2135 return result;
2136 }
2137 StatCache::getStatCacheData()->DelStat(nowcache);
2138
2139 // Make new directory object("dir/")
2140 if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts[0].tv_sec, ts[1].tv_sec, ts[0].tv_sec, stbuf.st_uid, stbuf.st_gid))){
2141 return result;
2142 }
2143 }else{
2144 // normal object or directory object of newer version
2145
2146 // open & load
2147 AutoFdEntity autoent;
2148 FdEntity* ent;
2149 if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
2150 S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
2151 return result;
2152 }
2153
2154 // set mtime/ctime
2155 if(0 != (result = ent->SetMCtime(ts[1], ts[0]))){
2156 S3FS_PRN_ERR("could not set mtime and ctime to file(%s): result=%d", strpath.c_str(), result);
2157 return result;
2158 }
2159
2160 // set atime
2161 if(0 != (result = ent->SetAtime(ts[0]))){
2162 S3FS_PRN_ERR("could not set atime to file(%s): result=%d", strpath.c_str(), result);
2163 return result;
2164 }
2165
2166 // upload
2167 if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
2168 S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
2169 return result;
2170 }
2171 StatCache::getStatCacheData()->DelStat(nowcache);
2172 }
2173 S3FS_MALLOCTRIM(0);
2174
2175 return result;
2176 }
2177
2178 static int s3fs_truncate(const char* _path, off_t size)
2179 {
2180 WTF8_ENCODE(path)
2181 int result;
2182 headers_t meta;
2183 AutoFdEntity autoent;
2184 FdEntity* ent = NULL;
2185
2186 S3FS_PRN_INFO("[path=%s][size=%lld]", path, static_cast<long long>(size));
2187
2188 if(size < 0){
2189 size = 0;
2190 }
2191
2192 if(0 != (result = check_parent_object_access(path, X_OK))){
2193 return result;
2194 }
2195 if(0 != (result = check_object_access(path, W_OK, NULL))){
2196 return result;
2197 }
2198
2199 // Get file information
2200 if(0 == (result = get_object_attribute(path, NULL, &meta))){
2201 // Exists -> Get file(with size)
2202 if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, false, true, AutoLock::NONE))){
2203 S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
2204 return -EIO;
2205 }
2206 if(0 != (result = ent->Load(0, size, AutoLock::NONE))){
2207 S3FS_PRN_ERR("could not download file(%s): result=%d", path, result);
2208 return result;
2209 }
2210
2211 ent->UpdateCtime();
2212 }else{
2213 // Not found -> Make tmpfile(with size)
2214
2215 struct fuse_context* pcxt;
2216 if(NULL == (pcxt = fuse_get_context())){
2217 return -EIO;
2218 }
2219 time_t now = time(NULL);
2220 meta["Content-Type"] = std::string("application/octet-stream"); // Static
2221 meta["x-amz-meta-mode"] = str(S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO);
2222 meta["x-amz-meta-ctime"] = str(now);
2223 meta["x-amz-meta-mtime"] = str(now);
2224 meta["x-amz-meta-uid"] = str(pcxt->uid);
2225 meta["x-amz-meta-gid"] = str(pcxt->gid);
2226
2227 if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, true, true, AutoLock::NONE))){
2228 S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
2229 return -EIO;
2230 }
2231 }
2232
2233 // upload
2234 if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
2235 S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
2236 return result;
2237 }
2238
2239 StatCache::getStatCacheData()->DelStat(path);
2240 S3FS_MALLOCTRIM(0);
2241
2242 return result;
2243 }
2244
2245 static int s3fs_open(const char* _path, struct fuse_file_info* fi)
2246 {
2247 WTF8_ENCODE(path)
2248 int result;
2249 struct stat st;
2250 bool needs_flush = false;
2251
2252 S3FS_PRN_INFO("[path=%s][flags=0x%x]", path, fi->flags);
2253
2254 if ((fi->flags & O_ACCMODE) == O_RDONLY && fi->flags & O_TRUNC) {
2255 return -EACCES;
2256 }
2257
2258 // [NOTE]
2259 // Delete the Stats cache only if the file is not open.
2260 // If the file is open, the stats cache will not be deleted as
2261 // there are cases where the object does not exist on the server
2262 // and only the Stats cache exists.
2263 //
2264 if(StatCache::getStatCacheData()->HasStat(path)){
2265 if(!FdManager::HasOpenEntityFd(path)){
2266 StatCache::getStatCacheData()->DelStat(path);
2267 }
2268 }
2269
2270 int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK);
2271 if(0 != (result = check_parent_object_access(path, X_OK))){
2272 return result;
2273 }
2274
2275 result = check_object_access(path, mask, &st);
2276 if(-ENOENT == result){
2277 if(0 != (result = check_parent_object_access(path, W_OK))){
2278 return result;
2279 }
2280 }else if(0 != result){
2281 return result;
2282 }
2283
2284 if((unsigned int)fi->flags & O_TRUNC){
2285 if(0 != st.st_size){
2286 st.st_size = 0;
2287 needs_flush = true;
2288 }
2289 }
2290 if(!S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)){
2291 st.st_mtime = -1;
2292 }
2293
2294 AutoFdEntity autoent;
2295 FdEntity* ent;
2296 headers_t meta;
2297 get_object_attribute(path, NULL, &meta, true, NULL, true); // no truncate cache
2298 if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, fi->flags, false, true, AutoLock::NONE))){
2299 StatCache::getStatCacheData()->DelStat(path);
2300 return -EIO;
2301 }
2302
2303 if (needs_flush){
2304 time_t now = time(NULL);
2305 struct timespec ts = {now, 0};
2306 ent->SetMCtime(ts, ts);
2307 if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), path, true))){
2308 S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
2309 StatCache::getStatCacheData()->DelStat(path);
2310 return result;
2311 }
2312 }
2313 fi->fh = autoent.Detach(); // KEEP fdentity open;
2314
2315 S3FS_MALLOCTRIM(0);
2316
2317 return 0;
2318 }
2319
2320 static int s3fs_read(const char* _path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi)
2321 {
2322 WTF8_ENCODE(path)
2323 ssize_t res;
2324
2325 S3FS_PRN_DBG("[path=%s][size=%zu][offset=%lld][pseudo_fd=%llu]", path, size, static_cast<long long>(offset), (unsigned long long)(fi->fh));
2326
2327 AutoFdEntity autoent;
2328 FdEntity* ent;
2329 if(NULL == (ent = autoent.GetExistFdEntity(path, static_cast<int>(fi->fh)))){
2330 S3FS_PRN_ERR("could not find opened pseudo_fd(=%llu) for path(%s)", (unsigned long long)(fi->fh), path);
2331 return -EIO;
2332 }
2333
2334 // check real file size
2335 off_t realsize = 0;
2336 if(!ent->GetSize(realsize) || 0 == realsize){
2337 S3FS_PRN_DBG("file size is 0, so break to read.");
2338 return 0;
2339 }
2340
2341 if(0 > (res = ent->Read(static_cast<int>(fi->fh), buf, offset, size, false))){
2342 S3FS_PRN_WARN("failed to read file(%s). result=%zd", path, res);
2343 }
2344
2345 return static_cast<int>(res);
2346 }
2347
2348 static int s3fs_write(const char* _path, const char* buf, size_t size, off_t offset, struct fuse_file_info* fi)
2349 {
2350 WTF8_ENCODE(path)
2351 ssize_t res;
2352
2353 S3FS_PRN_DBG("[path=%s][size=%zu][offset=%lld][pseudo_fd=%llu]", path, size, static_cast<long long int>(offset), (unsigned long long)(fi->fh));
2354
2355 AutoFdEntity autoent;
2356 FdEntity* ent;
2357 if(NULL == (ent = autoent.GetExistFdEntity(path, static_cast<int>(fi->fh)))){
2358 S3FS_PRN_ERR("could not find opened pseudo_fd(%llu) for path(%s)", (unsigned long long)(fi->fh), path);
2359 return -EIO;
2360 }
2361
2362 if(0 > (res = ent->Write(static_cast<int>(fi->fh), buf, offset, size))){
2363 S3FS_PRN_WARN("failed to write file(%s). result=%zd", path, res);
2364 }
2365
2366 if(max_dirty_data != -1 && ent->BytesModified() >= max_dirty_data){
2367 int flushres;
2368 if(0 != (flushres = ent->RowFlush(static_cast<int>(fi->fh), path, true))){
2369 S3FS_PRN_ERR("could not upload file(%s): result=%d", path, flushres);
2370 StatCache::getStatCacheData()->DelStat(path);
2371 return flushres;
2372 }
2373 // Punch a hole in the file to recover disk space.
2374 if(!ent->PunchHole()){
2375 S3FS_PRN_WARN("could not punching HOLEs to a cache file, but continue.");
2376 }
2377 }
2378
2379 return static_cast<int>(res);
2380 }
2381
2382 static int s3fs_statfs(const char* _path, struct statvfs* stbuf)
2383 {
2384 // WTF8_ENCODE(path)
2385 stbuf->f_bsize = 16 * 1024 * 1024;
2386 stbuf->f_blocks = static_cast<fsblkcnt_t>(~0) / stbuf->f_bsize;
2387 stbuf->f_bfree = stbuf->f_blocks;
2388 stbuf->f_bavail = stbuf->f_blocks;
2389 stbuf->f_namemax = NAME_MAX;
2390 return 0;
2391 }
2392
2393 static int s3fs_flush(const char* _path, struct fuse_file_info* fi)
2394 {
2395 WTF8_ENCODE(path)
2396 int result;
2397
2398 S3FS_PRN_INFO("[path=%s][pseudo_fd=%llu]", path, (unsigned long long)(fi->fh));
2399
2400 int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK);
2401 if(0 != (result = check_parent_object_access(path, X_OK))){
2402 return result;
2403 }
2404 result = check_object_access(path, mask, NULL);
2405 if(-ENOENT == result){
2406 if(0 != (result = check_parent_object_access(path, W_OK))){
2407 return result;
2408 }
2409 }else if(0 != result){
2410 return result;
2411 }
2412
2413 AutoFdEntity autoent;
2414 FdEntity* ent;
2415 if(NULL != (ent = autoent.GetExistFdEntity(path, static_cast<int>(fi->fh)))){
2416 ent->UpdateMtime(true); // clear the flag not to update mtime.
2417 ent->UpdateCtime();
2418 result = ent->Flush(static_cast<int>(fi->fh), false);
2419 StatCache::getStatCacheData()->DelStat(path);
2420 }
2421 S3FS_MALLOCTRIM(0);
2422
2423 return result;
2424 }
2425
2426 // [NOTICE]
2427 // Assumption is a valid fd.
2428 //
2429 static int s3fs_fsync(const char* _path, int datasync, struct fuse_file_info* fi)
2430 {
2431 WTF8_ENCODE(path)
2432 int result = 0;
2433
2434 S3FS_PRN_INFO("[path=%s][pseudo_fd=%llu]", path, (unsigned long long)(fi->fh));
2435
2436 AutoFdEntity autoent;
2437 FdEntity* ent;
2438 if(NULL != (ent = autoent.GetExistFdEntity(path, static_cast<int>(fi->fh)))){
2439 if(0 == datasync){
2440 ent->UpdateMtime();
2441 ent->UpdateCtime();
2442 }
2443 result = ent->Flush(static_cast<int>(fi->fh), false);
2444 }
2445 S3FS_MALLOCTRIM(0);
2446
2447 // Issue 320: Delete stat cache entry because st_size may have changed.
2448 StatCache::getStatCacheData()->DelStat(path);
2449
2450 return result;
2451 }
2452
2453 static int s3fs_release(const char* _path, struct fuse_file_info* fi)
2454 {
2455 WTF8_ENCODE(path)
2456 S3FS_PRN_INFO("[path=%s][pseudo_fd=%llu]", path, (unsigned long long)(fi->fh));
2457
2458 // [NOTE]
2459 // All opened file's stats is cached with no truncate flag.
2460 // Thus we unset it here.
2461 StatCache::getStatCacheData()->ChangeNoTruncateFlag(std::string(path), false);
2462
2463 // [NOTICE]
2464 // At first, we remove stats cache.
2465 // Because fuse does not wait for response from "release" function. :-(
2466 // And fuse runs next command before this function returns.
2467 // Thus we call deleting stats function ASAP.
2468 //
2469 if((fi->flags & O_RDWR) || (fi->flags & O_WRONLY)){
2470 StatCache::getStatCacheData()->DelStat(path);
2471 }
2472
2473 { // scope for AutoFdEntity
2474 AutoFdEntity autoent;
2475
2476 // [NOTE]
2477 // The pseudo fd stored in fi->fh is attached to AutoFdEntry so that it can be
2478 // destroyed here.
2479 //
2480 if(!autoent.Attach(path, static_cast<int>(fi->fh))){
2481 S3FS_PRN_ERR("could not find pseudo_fd(%llu) for path(%s)", (unsigned long long)(fi->fh), path);
2482 return -EIO;
2483 }
2484 }
2485
2486 // check - for debug
2487 if(S3fsLog::IsS3fsLogDbg()){
2488 if(FdManager::HasOpenEntityFd(path)){
2489 S3FS_PRN_WARN("file(%s) is still opened(another pseudo fd is opend).", path);
2490 }
2491 }
2492 S3FS_MALLOCTRIM(0);
2493
2494 return 0;
2495 }
2496
2497 static int s3fs_opendir(const char* _path, struct fuse_file_info* fi)
2498 {
2499 WTF8_ENCODE(path)
2500 int result;
2501 int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK);
2502
2503 S3FS_PRN_INFO("[path=%s][flags=0x%x]", path, fi->flags);
2504
2505 if(0 == (result = check_object_access(path, mask, NULL))){
2506 result = check_parent_object_access(path, X_OK);
2507 }
2508 S3FS_MALLOCTRIM(0);
2509
2510 return result;
2511 }
2512
2513 static bool multi_head_callback(S3fsCurl* s3fscurl)
2514 {
2515 if(!s3fscurl){
2516 return false;
2517 }
2518 std::string saved_path = s3fscurl->GetSpecialSavedPath();
2519 if(!StatCache::getStatCacheData()->AddStat(saved_path, *(s3fscurl->GetResponseHeaders()))){
2520 S3FS_PRN_ERR("failed adding stat cache [path=%s]", saved_path.c_str());
2521 return false;
2522 }
2523 return true;
2524 }
2525
2526 static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl)
2527 {
2528 if(!s3fscurl){
2529 return NULL;
2530 }
2531 size_t ssec_key_pos= s3fscurl->GetLastPreHeadSeecKeyPos();
2532 int retry_count = s3fscurl->GetMultipartRetryCount();
2533
2534 // retry next sse key.
2535 // if end of sse key, set retry master count is up.
2536 ssec_key_pos = (ssec_key_pos == static_cast<size_t>(-1) ? 0 : ssec_key_pos + 1);
2537 if(0 == S3fsCurl::GetSseKeyCount() || S3fsCurl::GetSseKeyCount() <= ssec_key_pos){
2538 if(s3fscurl->IsOverMultipartRetryCount()){
2539 S3FS_PRN_ERR("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpecialSavedPath().c_str());
2540 return NULL;
2541 }
2542 ssec_key_pos = -1;
2543 retry_count++;
2544 }
2545
2546 S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
2547 std::string path = s3fscurl->GetPath();
2548 std::string base_path = s3fscurl->GetBasePath();
2549 std::string saved_path = s3fscurl->GetSpecialSavedPath();
2550
2551 if(!newcurl->PreHeadRequest(path, base_path, saved_path, ssec_key_pos)){
2552 S3FS_PRN_ERR("Could not duplicate curl object(%s).", saved_path.c_str());
2553 delete newcurl;
2554 return NULL;
2555 }
2556 newcurl->SetMultipartRetryCount(retry_count);
2557
2558 return newcurl;
2559 }
2560
2561 static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler)
2562 {
2563 S3fsMultiCurl curlmulti(S3fsCurl::GetMaxMultiRequest());
2564 s3obj_list_t headlist;
2565 s3obj_list_t fillerlist;
2566 int result = 0;
2567
2568 S3FS_PRN_INFO1("[path=%s][list=%zu]", path, headlist.size());
2569
2570 // Make base path list.
2571 head.GetNameList(headlist, true, false); // get name with "/".
2572
2573 // Initialize S3fsMultiCurl
2574 curlmulti.SetSuccessCallback(multi_head_callback);
2575 curlmulti.SetRetryCallback(multi_head_retry_callback);
2576
2577 s3obj_list_t::iterator iter;
2578
2579 fillerlist.clear();
2580 // Make single head request(with max).
2581 for(iter = headlist.begin(); headlist.end() != iter; iter = headlist.erase(iter)){
2582 std::string disppath = path + (*iter);
2583 std::string etag = head.GetETag((*iter).c_str());
2584
2585 std::string fillpath = disppath;
2586 if('/' == *disppath.rbegin()){
2587 fillpath.erase(fillpath.length() -1);
2588 }
2589 fillerlist.push_back(fillpath);
2590
2591 if(StatCache::getStatCacheData()->HasStat(disppath, etag.c_str())){
2592 continue;
2593 }
2594
2595 // First check for directory, start checking "not SSE-C".
2596 // If checking failed, retry to check with "SSE-C" by retry callback func when SSE-C mode.
2597 S3fsCurl* s3fscurl = new S3fsCurl();
2598 if(!s3fscurl->PreHeadRequest(disppath, (*iter), disppath)){ // target path = cache key path.(ex "dir/")
2599 S3FS_PRN_WARN("Could not make curl object for head request(%s).", disppath.c_str());
2600 delete s3fscurl;
2601 continue;
2602 }
2603
2604 if(!curlmulti.SetS3fsCurlObject(s3fscurl)){
2605 S3FS_PRN_WARN("Could not make curl object into multi curl(%s).", disppath.c_str());
2606 delete s3fscurl;
2607 continue;
2608 }
2609 }
2610
2611 // Multi request
2612 if(0 != (result = curlmulti.Request())){
2613 // If result is -EIO, it is something error occurred.
2614 // This case includes that the object is encrypting(SSE) and s3fs does not have keys.
2615 // So s3fs set result to 0 in order to continue the process.
2616 if(-EIO == result){
2617 S3FS_PRN_WARN("error occurred in multi request(errno=%d), but continue...", result);
2618 result = 0;
2619 }else{
2620 S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result);
2621 return result;
2622 }
2623 }
2624
2625 // populate fuse buffer
2626 // here is best position, because a case is cache size < files in directory
2627 //
2628 for(iter = fillerlist.begin(); fillerlist.end() != iter; ++iter){
2629 struct stat st;
2630 bool in_cache = StatCache::getStatCacheData()->GetStat((*iter), &st);
2631 std::string bpath = mybasename((*iter));
2632 if(use_wtf8){
2633 bpath = s3fs_wtf8_decode(bpath);
2634 }
2635 if(in_cache){
2636 filler(buf, bpath.c_str(), &st, 0);
2637 }else{
2638 S3FS_PRN_INFO2("Could not find %s file in stat cache.", (*iter).c_str());
2639 filler(buf, bpath.c_str(), 0, 0);
2640 }
2641 }
2642
2643 return result;
2644 }
2645
2646 static int s3fs_readdir(const char* _path, void* buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info* fi)
2647 {
2648 WTF8_ENCODE(path)
2649 S3ObjList head;
2650 int result;
2651
2652 S3FS_PRN_INFO("[path=%s]", path);
2653
2654 if(0 != (result = check_object_access(path, R_OK, NULL))){
2655 return result;
2656 }
2657
2658 // get a list of all the objects
2659 if((result = list_bucket(path, head, "/")) != 0){
2660 S3FS_PRN_ERR("list_bucket returns error(%d).", result);
2661 return result;
2662 }
2663
2664 // force to add "." and ".." name.
2665 filler(buf, ".", 0, 0);
2666 filler(buf, "..", 0, 0);
2667 if(head.IsEmpty()){
2668 return 0;
2669 }
2670
2671 // Send multi head request for stats caching.
2672 std::string strpath = path;
2673 if(strcmp(path, "/") != 0){
2674 strpath += "/";
2675 }
2676 if(0 != (result = readdir_multi_head(strpath.c_str(), head, buf, filler))){
2677 S3FS_PRN_ERR("readdir_multi_head returns error(%d).", result);
2678 }
2679 S3FS_MALLOCTRIM(0);
2680
2681 return result;
2682 }
2683
2684 static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only)
2685 {
2686 std::string s3_realpath;
2687 std::string query_delimiter;
2688 std::string query_prefix;
2689 std::string query_maxkey;
2690 std::string next_continuation_token;
2691 std::string next_marker;
2692 bool truncated = true;
2693 S3fsCurl s3fscurl;
2694 xmlDocPtr doc;
2695
2696 S3FS_PRN_INFO1("[path=%s]", path);
2697
2698 if(delimiter && 0 < strlen(delimiter)){
2699 query_delimiter += "delimiter=";
2700 query_delimiter += delimiter;
2701 query_delimiter += "&";
2702 }
2703
2704 query_prefix += "&prefix=";
2705 s3_realpath = get_realpath(path);
2706 if(s3_realpath.empty() || '/' != *s3_realpath.rbegin()){
2707 // last word must be "/"
2708 query_prefix += urlEncode(s3_realpath.substr(1) + "/");
2709 }else{
2710 query_prefix += urlEncode(s3_realpath.substr(1));
2711 }
2712 if (check_content_only){
2713 // Just need to know if there are child objects in dir
2714 // For dir with children, expect "dir/" and "dir/child"
2715 query_maxkey += "max-keys=2";
2716 }else{
2717 query_maxkey += "max-keys=" + str(max_keys_list_object);
2718 }
2719
2720 while(truncated){
2721 // append parameters to query in alphabetical order
2722 std::string each_query;
2723 if(!next_continuation_token.empty()){
2724 each_query += "continuation-token=" + urlEncode(next_continuation_token) + "&";
2725 next_continuation_token = "";
2726 }
2727 each_query += query_delimiter;
2728 if(S3fsCurl::IsListObjectsV2()){
2729 each_query += "list-type=2&";
2730 }
2731 if(!next_marker.empty()){
2732 each_query += "marker=" + urlEncode(next_marker) + "&";
2733 next_marker = "";
2734 }
2735 each_query += query_maxkey;
2736 each_query += query_prefix;
2737
2738 // request
2739 int result;
2740 if(0 != (result = s3fscurl.ListBucketRequest(path, each_query.c_str()))){
2741 S3FS_PRN_ERR("ListBucketRequest returns with error.");
2742 return result;
2743 }
2744 BodyData* body = s3fscurl.GetBodyData();
2745
2746 // xmlDocPtr
2747 if(NULL == (doc = xmlReadMemory(body->str(), static_cast<int>(body->size()), "", NULL, 0))){
2748 S3FS_PRN_ERR("xmlReadMemory returns with error.");
2749 return -EIO;
2750 }
2751 if(0 != append_objects_from_xml(path, doc, head)){
2752 S3FS_PRN_ERR("append_objects_from_xml returns with error.");
2753 xmlFreeDoc(doc);
2754 return -EIO;
2755 }
2756 if(true == (truncated = is_truncated(doc))){
2757 xmlChar* tmpch;
2758 if(NULL != (tmpch = get_next_continuation_token(doc))){
2759 next_continuation_token = (char*)tmpch;
2760 xmlFree(tmpch);
2761 }else if(NULL != (tmpch = get_next_marker(doc))){
2762 next_marker = (char*)tmpch;
2763 xmlFree(tmpch);
2764 }
2765
2766 if(next_continuation_token.empty() && next_marker.empty()){
2767 // If did not specify "delimiter", s3 did not return "NextMarker".
2768 // On this case, can use last name for next marker.
2769 //
2770 std::string lastname;
2771 if(!head.GetLastName(lastname)){
2772 S3FS_PRN_WARN("Could not find next marker, thus break loop.");
2773 truncated = false;
2774 }else{
2775 next_marker = s3_realpath.substr(1);
2776 if(s3_realpath.empty() || '/' != *s3_realpath.rbegin()){
2777 next_marker += "/";
2778 }
2779 next_marker += lastname;
2780 }
2781 }
2782 }
2783 S3FS_XMLFREEDOC(doc);
2784
2785 // reset(initialize) curl object
2786 s3fscurl.DestroyCurlHandle();
2787
2788 if(check_content_only){
2789 break;
2790 }
2791 }
2792 S3FS_MALLOCTRIM(0);
2793
2794 return 0;
2795 }
2796
2797 static int remote_mountpath_exists(const char* path)
2798 {
2799 struct stat stbuf;
2800 int result;
2801
2802 S3FS_PRN_INFO1("[path=%s]", path);
2803
2804 // getattr will prefix the path with the remote mountpoint
2805 if(0 != (result = get_object_attribute("/", &stbuf, NULL))){
2806 return result;
2807 }
2808 if(!S_ISDIR(stbuf.st_mode)){
2809 return -ENOTDIR;
2810 }
2811 return 0;
2812 }
2813
2814
2815 static void free_xattrs(xattrs_t& xattrs)
2816 {
2817 for(xattrs_t::iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
2818 delete iter->second;
2819 }
2820 xattrs.clear();
2821 }
2822
2823 static bool parse_xattr_keyval(const std::string& xattrpair, std::string& key, PXATTRVAL& pval)
2824 {
2825 // parse key and value
2826 size_t pos;
2827 std::string tmpval;
2828 if(std::string::npos == (pos = xattrpair.find_first_of(':'))){
2829 S3FS_PRN_ERR("one of xattr pair(%s) is wrong format.", xattrpair.c_str());
2830 return false;
2831 }
2832 key = xattrpair.substr(0, pos);
2833 tmpval = xattrpair.substr(pos + 1);
2834
2835 if(!takeout_str_dquart(key) || !takeout_str_dquart(tmpval)){
2836 S3FS_PRN_ERR("one of xattr pair(%s) is wrong format.", xattrpair.c_str());
2837 return false;
2838 }
2839
2840 pval = new XATTRVAL;
2841 pval->length = 0;
2842 pval->pvalue = s3fs_decode64(tmpval.c_str(), &pval->length);
2843
2844 return true;
2845 }
2846
2847 static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs)
2848 {
2849 xattrs.clear();
2850
2851 // decode
2852 std::string jsonxattrs = urlDecode(strxattrs);
2853
2854 // get from "{" to "}"
2855 std::string restxattrs;
2856 {
2857 size_t startpos;
2858 size_t endpos = std::string::npos;
2859 if(std::string::npos != (startpos = jsonxattrs.find_first_of('{'))){
2860 endpos = jsonxattrs.find_last_of('}');
2861 }
2862 if(startpos == std::string::npos || endpos == std::string::npos || endpos <= startpos){
2863 S3FS_PRN_WARN("xattr header(%s) is not json format.", jsonxattrs.c_str());
2864 return 0;
2865 }
2866 restxattrs = jsonxattrs.substr(startpos + 1, endpos - (startpos + 1));
2867 }
2868
2869 // parse each key:val
2870 for(size_t pair_nextpos = restxattrs.find_first_of(','); !restxattrs.empty(); restxattrs = (pair_nextpos != std::string::npos ? restxattrs.substr(pair_nextpos + 1) : std::string("")), pair_nextpos = restxattrs.find_first_of(',')){
2871 std::string pair = pair_nextpos != std::string::npos ? restxattrs.substr(0, pair_nextpos) : restxattrs;
2872 std::string key;
2873 PXATTRVAL pval = NULL;
2874 if(!parse_xattr_keyval(pair, key, pval)){
2875 // something format error, so skip this.
2876 continue;
2877 }
2878 xattrs[key] = pval;
2879 }
2880 return xattrs.size();
2881 }
2882
2883 static std::string build_xattrs(const xattrs_t& xattrs)
2884 {
2885 std::string strxattrs("{");
2886
2887 bool is_set = false;
2888 for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
2889 if(is_set){
2890 strxattrs += ',';
2891 }else{
2892 is_set = true;
2893 }
2894 strxattrs += '\"';
2895 strxattrs += iter->first;
2896 strxattrs += "\":\"";
2897
2898 if(iter->second){
2899 char* base64val = s3fs_base64((iter->second)->pvalue, (iter->second)->length);
2900 if(base64val){
2901 strxattrs += base64val;
2902 delete[] base64val;
2903 }
2904 }
2905 strxattrs += '\"';
2906 }
2907 strxattrs += '}';
2908
2909 strxattrs = urlEncode(strxattrs);
2910
2911 return strxattrs;
2912 }
2913
2914 static int set_xattrs_to_header(headers_t& meta, const char* name, const char* value, size_t size, int flags)
2915 {
2916 std::string strxattrs;
2917 xattrs_t xattrs;
2918
2919 headers_t::iterator iter;
2920 if(meta.end() == (iter = meta.find("x-amz-meta-xattr"))){
2921 #if defined(XATTR_REPLACE)
2922 if(XATTR_REPLACE == (flags & XATTR_REPLACE)){
2923 // there is no xattr header but flags is replace, so failure.
2924 return -ENOATTR;
2925 }
2926 #endif
2927 }else{
2928 #if defined(XATTR_CREATE)
2929 if(XATTR_CREATE == (flags & XATTR_CREATE)){
2930 // found xattr header but flags is only creating, so failure.
2931 return -EEXIST;
2932 }
2933 #endif
2934 strxattrs = iter->second;
2935 }
2936
2937 // get map as xattrs_t
2938 parse_xattrs(strxattrs, xattrs);
2939
2940 // add name(do not care overwrite and empty name/value)
2941 xattrs_t::iterator xiter;
2942 if(xattrs.end() != (xiter = xattrs.find(std::string(name)))){
2943 // found same head. free value.
2944 delete xiter->second;
2945 }
2946
2947 PXATTRVAL pval = new XATTRVAL;
2948 pval->length = size;
2949 if(0 < size){
2950 pval->pvalue = new unsigned char[size];
2951 memcpy(pval->pvalue, value, size);
2952 }else{
2953 pval->pvalue = NULL;
2954 }
2955 xattrs[std::string(name)] = pval;
2956
2957 // build new strxattrs(not encoded) and set it to headers_t
2958 meta["x-amz-meta-xattr"] = build_xattrs(xattrs);
2959
2960 free_xattrs(xattrs);
2961
2962 return 0;
2963 }
2964
2965 #if defined(__APPLE__)
2966 static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags, uint32_t position)
2967 #else
2968 static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags)
2969 #endif
2970 {
2971 S3FS_PRN_INFO("[path=%s][name=%s][value=%p][size=%zu][flags=0x%x]", path, name, value, size, flags);
2972
2973 if((value && 0 == size) || (!value && 0 < size)){
2974 S3FS_PRN_ERR("Wrong parameter: value(%p), size(%zu)", value, size);
2975 return 0;
2976 }
2977
2978 #if defined(__APPLE__)
2979 if (position != 0) {
2980 // No resource fork support
2981 return -EINVAL;
2982 }
2983 #endif
2984
2985 int result;
2986 std::string strpath;
2987 std::string newpath;
2988 std::string nowcache;
2989 headers_t meta;
2990 struct stat stbuf;
2991 dirtype nDirType = DIRTYPE_UNKNOWN;
2992
2993 if(0 == strcmp(path, "/")){
2994 S3FS_PRN_ERR("Could not change mode for mount point.");
2995 return -EIO;
2996 }
2997 if(0 != (result = check_parent_object_access(path, X_OK))){
2998 return result;
2999 }
3000 if(0 != (result = check_object_owner(path, &stbuf))){
3001 return result;
3002 }
3003
3004 if(S_ISDIR(stbuf.st_mode)){
3005 result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
3006 }else{
3007 strpath = path;
3008 nowcache = strpath;
3009 result = get_object_attribute(strpath.c_str(), NULL, &meta);
3010 }
3011 if(0 != result){
3012 return result;
3013 }
3014
3015 if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
3016 // Should rebuild directory object(except new type)
3017 // Need to remove old dir("dir" etc) and make new dir("dir/")
3018
3019 // At first, remove directory old object
3020 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
3021 return result;
3022 }
3023 StatCache::getStatCacheData()->DelStat(nowcache);
3024
3025 // Make new directory object("dir/")
3026 if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, stbuf.st_ctime, stbuf.st_uid, stbuf.st_gid))){
3027 return result;
3028 }
3029
3030 // need to set xattr header for directory.
3031 strpath = newpath;
3032 nowcache = strpath;
3033 }
3034
3035 // set xattr all object
3036 headers_t updatemeta;
3037 updatemeta["x-amz-meta-ctime"] = str(time(NULL));
3038 updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
3039 updatemeta["x-amz-metadata-directive"] = "REPLACE";
3040
3041 // check opened file handle.
3042 //
3043 // If the file starts uploading by multipart when the disk capacity is insufficient,
3044 // we need to put these header after finishing upload.
3045 // Or if the file is only open, we must update to FdEntity's internal meta.
3046 //
3047 AutoFdEntity autoent;
3048 FdEntity* ent;
3049 bool need_put_header = true;
3050 if(NULL != (ent = autoent.OpenExistFdEntity(path))){
3051 // get xattr and make new xattr
3052 std::string strxattr;
3053 if(ent->GetXattr(strxattr)){
3054 updatemeta["x-amz-meta-xattr"] = strxattr;
3055 }else{
3056 // [NOTE]
3057 // Set an empty xattr.
3058 // This requires the key to be present in order to add xattr.
3059 ent->SetXattr(strxattr);
3060 }
3061 if(0 != (result = set_xattrs_to_header(updatemeta, name, value, size, flags))){
3062 return result;
3063 }
3064
3065 if(ent->MergeOrgMeta(updatemeta)){
3066 // meta is changed, but now uploading.
3067 // then the meta is pending and accumulated to be put after the upload is complete.
3068 S3FS_PRN_INFO("meta pending until upload is complete");
3069 need_put_header = false;
3070
3071 // If there is data in the Stats cache, update the Stats cache.
3072 StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta);
3073 }
3074 }
3075 if(need_put_header){
3076 // not found opened file.
3077 if(0 != (result = set_xattrs_to_header(meta, name, value, size, flags))){
3078 return result;
3079 }
3080 merge_headers(meta, updatemeta, true);
3081
3082 // upload meta directly.
3083 if(0 != (result = put_headers(strpath.c_str(), meta, true))){
3084 return result;
3085 }
3086 StatCache::getStatCacheData()->DelStat(nowcache);
3087 }
3088
3089 return 0;
3090 }
3091
3092 #if defined(__APPLE__)
3093 static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size, uint32_t position)
3094 #else
3095 static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size)
3096 #endif
3097 {
3098 S3FS_PRN_INFO("[path=%s][name=%s][value=%p][size=%zu]", path, name, value, size);
3099
3100 if(!path || !name){
3101 return -EIO;
3102 }
3103
3104 #if defined(__APPLE__)
3105 if (position != 0) {
3106 // No resource fork support
3107 return -EINVAL;
3108 }
3109 #endif
3110
3111 int result;
3112 headers_t meta;
3113 xattrs_t xattrs;
3114
3115 // check parent directory attribute.
3116 if(0 != (result = check_parent_object_access(path, X_OK))){
3117 return result;
3118 }
3119
3120 // get headers
3121 if(0 != (result = get_object_attribute(path, NULL, &meta))){
3122 return result;
3123 }
3124
3125 // get xattrs
3126 headers_t::iterator hiter = meta.find("x-amz-meta-xattr");
3127 if(meta.end() == hiter){
3128 // object does not have xattrs
3129 return -ENOATTR;
3130 }
3131 std::string strxattrs = hiter->second;
3132
3133 parse_xattrs(strxattrs, xattrs);
3134
3135 // search name
3136 std::string strname = name;
3137 xattrs_t::iterator xiter = xattrs.find(strname);
3138 if(xattrs.end() == xiter){
3139 // not found name in xattrs
3140 free_xattrs(xattrs);
3141 return -ENOATTR;
3142 }
3143
3144 // decode
3145 size_t length = 0;
3146 unsigned char* pvalue = NULL;
3147 if(NULL != xiter->second){
3148 length = xiter->second->length;
3149 pvalue = xiter->second->pvalue;
3150 }
3151
3152 if(0 < size){
3153 if(static_cast<size_t>(size) < length){
3154 // over buffer size
3155 free_xattrs(xattrs);
3156 return -ERANGE;
3157 }
3158 if(pvalue){
3159 memcpy(value, pvalue, length);
3160 }
3161 }
3162 free_xattrs(xattrs);
3163
3164 return static_cast<int>(length);
3165 }
3166
3167 static int s3fs_listxattr(const char* path, char* list, size_t size)
3168 {
3169 S3FS_PRN_INFO("[path=%s][list=%p][size=%zu]", path, list, size);
3170
3171 if(!path){
3172 return -EIO;
3173 }
3174
3175 int result;
3176 headers_t meta;
3177 xattrs_t xattrs;
3178
3179 // check parent directory attribute.
3180 if(0 != (result = check_parent_object_access(path, X_OK))){
3181 return result;
3182 }
3183
3184 // get headers
3185 if(0 != (result = get_object_attribute(path, NULL, &meta))){
3186 return result;
3187 }
3188
3189 // get xattrs
3190 headers_t::iterator iter;
3191 if(meta.end() == (iter = meta.find("x-amz-meta-xattr"))){
3192 // object does not have xattrs
3193 return 0;
3194 }
3195 std::string strxattrs = iter->second;
3196
3197 parse_xattrs(strxattrs, xattrs);
3198
3199 // calculate total name length
3200 size_t total = 0;
3201 for(xattrs_t::const_iterator xiter = xattrs.begin(); xiter != xattrs.end(); ++xiter){
3202 if(!xiter->first.empty()){
3203 total += xiter->first.length() + 1;
3204 }
3205 }
3206
3207 if(0 == total){
3208 free_xattrs(xattrs);
3209 return 0;
3210 }
3211
3212 // check parameters
3213 if(0 == size){
3214 free_xattrs(xattrs);
3215 return static_cast<int>(total);
3216 }
3217 if(!list || size < total){
3218 free_xattrs(xattrs);
3219 return -ERANGE;
3220 }
3221
3222 // copy to list
3223 char* setpos = list;
3224 for(xattrs_t::const_iterator xiter = xattrs.begin(); xiter != xattrs.end(); ++xiter){
3225 if(!xiter->first.empty()){
3226 strcpy(setpos, xiter->first.c_str());
3227 setpos = &setpos[strlen(setpos) + 1];
3228 }
3229 }
3230 free_xattrs(xattrs);
3231
3232 return static_cast<int>(total);
3233 }
3234
3235 static int s3fs_removexattr(const char* path, const char* name)
3236 {
3237 S3FS_PRN_INFO("[path=%s][name=%s]", path, name);
3238
3239 if(!path || !name){
3240 return -EIO;
3241 }
3242
3243 int result;
3244 std::string strpath;
3245 std::string newpath;
3246 std::string nowcache;
3247 headers_t meta;
3248 xattrs_t xattrs;
3249 struct stat stbuf;
3250 dirtype nDirType = DIRTYPE_UNKNOWN;
3251
3252 if(0 == strcmp(path, "/")){
3253 S3FS_PRN_ERR("Could not change mode for mount point.");
3254 return -EIO;
3255 }
3256 if(0 != (result = check_parent_object_access(path, X_OK))){
3257 return result;
3258 }
3259 if(0 != (result = check_object_owner(path, &stbuf))){
3260 return result;
3261 }
3262
3263 if(S_ISDIR(stbuf.st_mode)){
3264 result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
3265 }else{
3266 strpath = path;
3267 nowcache = strpath;
3268 result = get_object_attribute(strpath.c_str(), NULL, &meta);
3269 }
3270 if(0 != result){
3271 return result;
3272 }
3273
3274 // get xattrs
3275 headers_t::iterator hiter = meta.find("x-amz-meta-xattr");
3276 if(meta.end() == hiter){
3277 // object does not have xattrs
3278 return -ENOATTR;
3279 }
3280 std::string strxattrs = hiter->second;
3281
3282 parse_xattrs(strxattrs, xattrs);
3283
3284 // check name xattrs
3285 std::string strname = name;
3286 xattrs_t::iterator xiter = xattrs.find(strname);
3287 if(xattrs.end() == xiter){
3288 free_xattrs(xattrs);
3289 return -ENOATTR;
3290 }
3291
3292 // make new header_t after deleting name xattr
3293 delete xiter->second;
3294 xattrs.erase(xiter);
3295
3296 if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
3297 // Should rebuild directory object(except new type)
3298 // Need to remove old dir("dir" etc) and make new dir("dir/")
3299
3300 // At first, remove directory old object
3301 if(0 != (result = remove_old_type_dir(strpath, nDirType))){
3302 return result;
3303 }
3304 StatCache::getStatCacheData()->DelStat(nowcache);
3305
3306 // Make new directory object("dir/")
3307 if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, stbuf.st_ctime, stbuf.st_uid, stbuf.st_gid))){
3308 free_xattrs(xattrs);
3309 return result;
3310 }
3311
3312 // need to set xattr header for directory.
3313 strpath = newpath;
3314 nowcache = strpath;
3315 }
3316
3317 // set xattr all object
3318 headers_t updatemeta;
3319 updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
3320 updatemeta["x-amz-metadata-directive"] = "REPLACE";
3321 if(!xattrs.empty()){
3322 updatemeta["x-amz-meta-xattr"] = build_xattrs(xattrs);
3323 }else{
3324 updatemeta["x-amz-meta-xattr"] = std::string(""); // This is a special case. If empty, this header will eventually be removed.
3325 }
3326 free_xattrs(xattrs);
3327
3328 // check opened file handle.
3329 //
3330 // If the file starts uploading by multipart when the disk capacity is insufficient,
3331 // we need to put these header after finishing upload.
3332 // Or if the file is only open, we must update to FdEntity's internal meta.
3333 //
3334 AutoFdEntity autoent;
3335 FdEntity* ent;
3336 bool need_put_header = true;
3337 if(NULL != (ent = autoent.OpenExistFdEntity(path))){
3338 if(ent->MergeOrgMeta(updatemeta)){
3339 // meta is changed, but now uploading.
3340 // then the meta is pending and accumulated to be put after the upload is complete.
3341 S3FS_PRN_INFO("meta pending until upload is complete");
3342 need_put_header = false;
3343
3344 // If there is data in the Stats cache, update the Stats cache.
3345 StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta);
3346 }
3347 }
3348 if(need_put_header){
3349 // not found opened file.
3350 if(updatemeta["x-amz-meta-xattr"].empty()){
3351 updatemeta.erase("x-amz-meta-xattr");
3352 }
3353
3354 merge_headers(meta, updatemeta, true);
3355
3356 // upload meta directly.
3357 if(0 != (result = put_headers(strpath.c_str(), meta, true))){
3358 return result;
3359 }
3360 StatCache::getStatCacheData()->DelStat(nowcache);
3361 }
3362
3363 return 0;
3364 }
3365
3366 // s3fs_init calls this function to exit cleanly from the fuse event loop.
3367 //
3368 // There's no way to pass an exit status to the high-level event loop API, so
3369 // this function stores the exit value in a global for main()
3370 static void s3fs_exit_fuseloop(int exit_status)
3371 {
3372 S3FS_PRN_ERR("Exiting FUSE event loop due to errors\n");
3373 s3fs_init_deferred_exit_status = exit_status;
3374 struct fuse_context *ctx = fuse_get_context();
3375 if (NULL != ctx) {
3376 fuse_exit(ctx->fuse);
3377 }
3378 }
3379
3380 static void* s3fs_init(struct fuse_conn_info* conn)
3381 {
3382 S3FS_PRN_INIT_INFO("init v%s(commit:%s) with %s", VERSION, COMMIT_HASH_VAL, s3fs_crypt_lib_name());
3383
3384 // cache(remove cache dirs at first)
3385 if(is_remove_cache && (!CacheFileStat::DeleteCacheFileStatDirectory() || !FdManager::DeleteCacheDirectory())){
3386 S3FS_PRN_DBG("Could not initialize cache directory.");
3387 }
3388
3389 // check loading IAM role name
3390 if(load_iamrole){
3391 // load IAM role name from http://169.254.169.254/latest/meta-data/iam/security-credentials
3392 //
3393 S3fsCurl s3fscurl;
3394 if(!s3fscurl.LoadIAMRoleFromMetaData()){
3395 S3FS_PRN_CRIT("could not load IAM role name from meta data.");
3396 s3fs_exit_fuseloop(EXIT_FAILURE);
3397 return NULL;
3398 }
3399 S3FS_PRN_INFO("loaded IAM role name = %s", S3fsCurl::GetIAMRole());
3400 }
3401
3402 if (create_bucket){
3403 int result = do_create_bucket();
3404 if(result != 0){
3405 s3fs_exit_fuseloop(result);
3406 return NULL;
3407 }
3408 }
3409
3410 // Check Bucket
3411 {
3412 int result;
3413 if(EXIT_SUCCESS != (result = s3fs_check_service())){
3414 s3fs_exit_fuseloop(result);
3415 return NULL;
3416 }
3417 }
3418
3419 // Investigate system capabilities
3420 #ifndef __APPLE__
3421 if((unsigned int)conn->capable & FUSE_CAP_ATOMIC_O_TRUNC){
3422 conn->want |= FUSE_CAP_ATOMIC_O_TRUNC;
3423 }
3424 #endif
3425
3426 if((unsigned int)conn->capable & FUSE_CAP_BIG_WRITES){
3427 conn->want |= FUSE_CAP_BIG_WRITES;
3428 }
3429
3430 // Signal object
3431 if(!S3fsSignals::Initialize()){
3432 S3FS_PRN_ERR("Failed to initialize signal object, but continue...");
3433 }
3434
3435 return NULL;
3436 }
3437
3438 static void s3fs_destroy(void*)
3439 {
3440 S3FS_PRN_INFO("destroy");
3441
3442 // Signal object
3443 if(!S3fsSignals::Destroy()){
3444 S3FS_PRN_WARN("Failed to clean up signal object.");
3445 }
3446
3447 // cache(remove at last)
3448 if(is_remove_cache && (!CacheFileStat::DeleteCacheFileStatDirectory() || !FdManager::DeleteCacheDirectory())){
3449 S3FS_PRN_WARN("Could not remove cache directory.");
3450 }
3451 }
3452
3453 static int s3fs_access(const char* path, int mask)
3454 {
3455 S3FS_PRN_INFO("[path=%s][mask=%s%s%s%s]", path,
3456 ((mask & R_OK) == R_OK) ? "R_OK " : "",
3457 ((mask & W_OK) == W_OK) ? "W_OK " : "",
3458 ((mask & X_OK) == X_OK) ? "X_OK " : "",
3459 (mask == F_OK) ? "F_OK" : "");
3460
3461 int result = check_object_access(path, mask, NULL);
3462 S3FS_MALLOCTRIM(0);
3463 return result;
3464 }
3465
3466 //
3467 // If calling with wrong region, s3fs gets following error body as 400 error code.
3468 // "<Error>
3469 // <Code>AuthorizationHeaderMalformed</Code>
3470 // <Message>The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1'</Message>
3471 // <Region>ap-northeast-1</Region>
3472 // <RequestId>...</RequestId>
3473 // <HostId>...</HostId>
3474 // </Error>"
3475 //
3476 // So this is cheap code but s3fs should get correct region automatically.
3477 //
3478 static bool check_region_error(const char* pbody, size_t len, std::string& expectregion)
3479 {
3480 if(!pbody){
3481 return false;
3482 }
3483
3484 std::string code;
3485 if(!simple_parse_xml(pbody, len, "Code", code) || code != "AuthorizationHeaderMalformed"){
3486 return false;
3487 }
3488
3489 if(!simple_parse_xml(pbody, len, "Region", expectregion)){
3490 return false;
3491 }
3492
3493 return true;
3494 }
3495
3496 static bool check_endpoint_error(const char* pbody, size_t len, std::string& expectendpoint)
3497 {
3498 if(!pbody){
3499 return false;
3500 }
3501
3502 std::string code;
3503 if(!simple_parse_xml(pbody, len, "Code", code) || code != "PermanentRedirect"){
3504 return false;
3505 }
3506
3507 if(!simple_parse_xml(pbody, len, "Endpoint", expectendpoint)){
3508 return false;
3509 }
3510
3511 return true;
3512 }
3513
3514 static int s3fs_check_service()
3515 {
3516 S3FS_PRN_INFO("check services.");
3517
3518 // At first time for access S3, we check IAM role if it sets.
3519 if(!S3fsCurl::CheckIAMCredentialUpdate()){
3520 S3FS_PRN_CRIT("Failed to check IAM role name(%s).", S3fsCurl::GetIAMRole());
3521 return EXIT_FAILURE;
3522 }
3523
3524 S3fsCurl s3fscurl;
3525 int res;
3526 if(0 > (res = s3fscurl.CheckBucket())){
3527 // get response code
3528 long responseCode = s3fscurl.GetLastResponseCode();
3529
3530 // check wrong endpoint, and automatically switch endpoint
3531 if(300 <= responseCode && responseCode < 500){
3532
3533 // check region error(for putting message or retrying)
3534 BodyData* body = s3fscurl.GetBodyData();
3535 std::string expectregion;
3536 std::string expectendpoint;
3537 if(check_region_error(body->str(), body->size(), expectregion)){
3538 // [NOTE]
3539 // If endpoint is not specified(using us-east-1 region) and
3540 // an error is encountered accessing a different region, we
3541 // will retry the check on the expected region.
3542 // see) https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro
3543 //
3544 if(is_specified_endpoint){
3545 const char* tmp_expect_ep = expectregion.c_str();
3546 S3FS_PRN_CRIT("The bucket region is not '%s', it is correctly '%s'. You should specify 'endpoint=%s' option.", endpoint.c_str(), tmp_expect_ep, tmp_expect_ep);
3547
3548 }else{
3549 // current endpoint is wrong, so try to connect to expected region.
3550 S3FS_PRN_CRIT("Failed to connect region '%s'(default), so retry to connect region '%s'.", endpoint.c_str(), expectregion.c_str());
3551 endpoint = expectregion;
3552 if(S3fsCurl::GetSignatureType() == V4_ONLY ||
3553 S3fsCurl::GetSignatureType() == V2_OR_V4){
3554 if(s3host == "http://s3.amazonaws.com"){
3555 s3host = "http://s3-" + endpoint + ".amazonaws.com";
3556 }else if(s3host == "https://s3.amazonaws.com"){
3557 s3host = "https://s3-" + endpoint + ".amazonaws.com";
3558 }
3559 }
3560
3561 // retry to check with new endpoint
3562 s3fscurl.DestroyCurlHandle();
3563 res = s3fscurl.CheckBucket();
3564 responseCode = s3fscurl.GetLastResponseCode();
3565 }
3566 }else if(check_endpoint_error(body->str(), body->size(), expectendpoint)){
3567 S3FS_PRN_ERR("S3 service returned PermanentRedirect with endpoint: %s", expectendpoint.c_str());
3568 return EXIT_FAILURE;
3569 }
3570 }
3571
3572 // try signature v2
3573 if(0 > res && (responseCode == 400 || responseCode == 403) && S3fsCurl::GetSignatureType() == V2_OR_V4){
3574 // switch sigv2
3575 S3FS_PRN_CRIT("Failed to connect by sigv4, so retry to connect by signature version 2.");
3576 S3fsCurl::SetSignatureType(V2_ONLY);
3577
3578 // retry to check with sigv2
3579 s3fscurl.DestroyCurlHandle();
3580 res = s3fscurl.CheckBucket();
3581 responseCode = s3fscurl.GetLastResponseCode();
3582 }
3583
3584 // check errors(after retrying)
3585 if(0 > res && responseCode != 200 && responseCode != 301){
3586 if(responseCode == 400){
3587 S3FS_PRN_CRIT("Bad Request(host=%s) - result of checking service.", s3host.c_str());
3588
3589 }else if(responseCode == 403){
3590 S3FS_PRN_CRIT("invalid credentials(host=%s) - result of checking service.", s3host.c_str());
3591
3592 }else if(responseCode == 404){
3593 S3FS_PRN_CRIT("bucket or key not found(host=%s) - result of checking service.", s3host.c_str());
3594
3595 }else{
3596 // another error
3597 S3FS_PRN_CRIT("unable to connect(host=%s) - result of checking service.", s3host.c_str());
3598 }
3599 return EXIT_FAILURE;
3600 }
3601 }
3602 s3fscurl.DestroyCurlHandle();
3603
3604 // make sure remote mountpath exists and is a directory
3605 if(!mount_prefix.empty()){
3606 if(remote_mountpath_exists(mount_prefix.c_str()) != 0){
3607 S3FS_PRN_CRIT("remote mountpath %s not found.", mount_prefix.c_str());
3608 return EXIT_FAILURE;
3609 }
3610 }
3611 S3FS_MALLOCTRIM(0);
3612
3613 return EXIT_SUCCESS;
3614 }
3615
3616 //
3617 // Read and Parse passwd file
3618 //
3619 // The line of the password file is one of the following formats:
3620 // (1) "accesskey:secretkey" : AWS format for default(all) access key/secret key
3621 // (2) "bucket:accesskey:secretkey" : AWS format for bucket's access key/secret key
3622 // (3) "key=value" : Content-dependent KeyValue contents
3623 //
3624 // This function sets result into bucketkvmap_t, it bucket name and key&value mapping.
3625 // If bucket name is empty(1 or 3 format), bucket name for mapping is set "\t" or "".
3626 //
3627 // Return: 1 - OK(could parse and set mapping etc.)
3628 // 0 - NG(could not read any value)
3629 // -1 - Should shutdown immediately
3630 //
3631 static int parse_passwd_file(bucketkvmap_t& resmap)
3632 {
3633 std::string line;
3634 size_t first_pos;
3635 readline_t linelist;
3636 readline_t::iterator iter;
3637
3638 // open passwd file
3639 std::ifstream PF(passwd_file.c_str());
3640 if(!PF.good()){
3641 S3FS_PRN_EXIT("could not open passwd file : %s", passwd_file.c_str());
3642 return -1;
3643 }
3644
3645 // read each line
3646 while(getline(PF, line)){
3647 line = trim(line);
3648 if(line.empty()){
3649 continue;
3650 }
3651 if('#' == line[0]){
3652 continue;
3653 }
3654 if(std::string::npos != line.find_first_of(" \t")){
3655 S3FS_PRN_EXIT("invalid line in passwd file, found whitespace character.");
3656 return -1;
3657 }
3658 if('[' == line[0]){
3659 S3FS_PRN_EXIT("invalid line in passwd file, found a bracket \"[\" character.");
3660 return -1;
3661 }
3662 linelist.push_back(line);
3663 }
3664
3665 // read '=' type
3666 kvmap_t kv;
3667 for(iter = linelist.begin(); iter != linelist.end(); ++iter){
3668 first_pos = iter->find_first_of('=');
3669 if(first_pos == std::string::npos){
3670 continue;
3671 }
3672 // formatted by "key=val"
3673 std::string key = trim(iter->substr(0, first_pos));
3674 std::string val = trim(iter->substr(first_pos + 1, std::string::npos));
3675 if(key.empty()){
3676 continue;
3677 }
3678 if(kv.end() != kv.find(key)){
3679 S3FS_PRN_WARN("same key name(%s) found in passwd file, skip this.", key.c_str());
3680 continue;
3681 }
3682 kv[key] = val;
3683 }
3684 // set special key name
3685 resmap[KEYVAL_FIELDS_TYPE] = kv;
3686
3687 // read ':' type
3688 for(iter = linelist.begin(); iter != linelist.end(); ++iter){
3689 first_pos = iter->find_first_of(':');
3690 size_t last_pos = iter->find_last_of(':');
3691 if(first_pos == std::string::npos){
3692 continue;
3693 }
3694 std::string bucketname;
3695 std::string accesskey;
3696 std::string secret;
3697 if(first_pos != last_pos){
3698 // formatted by "bucket:accesskey:secretkey"
3699 bucketname = trim(iter->substr(0, first_pos));
3700 accesskey = trim(iter->substr(first_pos + 1, last_pos - first_pos - 1));
3701 secret = trim(iter->substr(last_pos + 1, std::string::npos));
3702 }else{
3703 // formatted by "accesskey:secretkey"
3704 bucketname = ALLBUCKET_FIELDS_TYPE;
3705 accesskey = trim(iter->substr(0, first_pos));
3706 secret = trim(iter->substr(first_pos + 1, std::string::npos));
3707 }
3708 if(resmap.end() != resmap.find(bucketname)){
3709 S3FS_PRN_EXIT("there are multiple entries for the same bucket(%s) in the passwd file.", (bucketname.empty() ? "default" : bucketname.c_str()));
3710 return -1;
3711 }
3712 kv.clear();
3713 kv[AWS_ACCESSKEYID] = accesskey;
3714 kv[AWS_SECRETKEY] = secret;
3715 resmap[bucketname] = kv;
3716 }
3717 return (resmap.empty() ? 0 : 1);
3718 }
3719
3720 //
3721 // Return: 1 - OK(could read and set accesskey etc.)
3722 // 0 - NG(could not read)
3723 // -1 - Should shutdown immediately
3724 //
3725 static int check_for_aws_format(const kvmap_t& kvmap)
3726 {
3727 std::string str1(AWS_ACCESSKEYID);
3728 std::string str2(AWS_SECRETKEY);
3729
3730 if(kvmap.empty()){
3731 return 0;
3732 }
3733 kvmap_t::const_iterator str1_it = kvmap.find(str1);
3734 kvmap_t::const_iterator str2_it = kvmap.find(str2);
3735 if(kvmap.end() == str1_it && kvmap.end() == str2_it){
3736 return 0;
3737 }
3738 if(kvmap.end() == str1_it || kvmap.end() == str2_it){
3739 S3FS_PRN_EXIT("AWSAccesskey or AWSSecretkey is not specified.");
3740 return -1;
3741 }
3742 if(!S3fsCurl::SetAccessKey(str1_it->second.c_str(), str2_it->second.c_str())){
3743 S3FS_PRN_EXIT("failed to set access key/secret key.");
3744 return -1;
3745 }
3746 return 1;
3747 }
3748
3749 //
3750 // check_passwd_file_perms
3751 //
3752 // expect that global passwd_file variable contains
3753 // a non-empty value and is readable by the current user
3754 //
3755 // Check for too permissive access to the file
3756 // help save users from themselves via a security hole
3757 //
3758 // only two options: return or error out
3759 //
3760 static int check_passwd_file_perms()
3761 {
3762 struct stat info;
3763
3764 // let's get the file info
3765 if(stat(passwd_file.c_str(), &info) != 0){
3766 S3FS_PRN_EXIT("unexpected error from stat(%s).", passwd_file.c_str());
3767 return EXIT_FAILURE;
3768 }
3769
3770 // return error if any file has others permissions
3771 if( (info.st_mode & S_IROTH) ||
3772 (info.st_mode & S_IWOTH) ||
3773 (info.st_mode & S_IXOTH)) {
3774 S3FS_PRN_EXIT("credentials file %s should not have others permissions.", passwd_file.c_str());
3775 return EXIT_FAILURE;
3776 }
3777
3778 // Any local file should not have any group permissions
3779 // /etc/passwd-s3fs can have group permissions
3780 if(passwd_file != "/etc/passwd-s3fs"){
3781 if( (info.st_mode & S_IRGRP) ||
3782 (info.st_mode & S_IWGRP) ||
3783 (info.st_mode & S_IXGRP)) {
3784 S3FS_PRN_EXIT("credentials file %s should not have group permissions.", passwd_file.c_str());
3785 return EXIT_FAILURE;
3786 }
3787 }else{
3788 // "/etc/passwd-s3fs" does not allow group write.
3789 if((info.st_mode & S_IWGRP)){
3790 S3FS_PRN_EXIT("credentials file %s should not have group writable permissions.", passwd_file.c_str());
3791 return EXIT_FAILURE;
3792 }
3793 }
3794 if((info.st_mode & S_IXUSR) || (info.st_mode & S_IXGRP)){
3795 S3FS_PRN_EXIT("credentials file %s should not have executable permissions.", passwd_file.c_str());
3796 return EXIT_FAILURE;
3797 }
3798 return EXIT_SUCCESS;
3799 }
3800
3801 static int read_aws_credentials_file(const std::string &filename)
3802 {
3803 // open passwd file
3804 std::ifstream PF(filename.c_str());
3805 if(!PF.good()){
3806 return -1;
3807 }
3808
3809 std::string profile;
3810 std::string accesskey;
3811 std::string secret;
3812 std::string session_token;
3813
3814 // read each line
3815 std::string line;
3816 while(getline(PF, line)){
3817 line = trim(line);
3818 if(line.empty()){
3819 continue;
3820 }
3821 if('#' == line[0]){
3822 continue;
3823 }
3824
3825 if(line.size() > 2 && line[0] == '[' && line[line.size() - 1] == ']') {
3826 if(profile == aws_profile){
3827 break;
3828 }
3829 profile = line.substr(1, line.size() - 2);
3830 accesskey.clear();
3831 secret.clear();
3832 session_token.clear();
3833 }
3834
3835 size_t pos = line.find_first_of('=');
3836 if(pos == std::string::npos){
3837 continue;
3838 }
3839 std::string key = trim(line.substr(0, pos));
3840 std::string value = trim(line.substr(pos + 1, std::string::npos));
3841 if(key == "aws_access_key_id"){
3842 accesskey = value;
3843 }else if(key == "aws_secret_access_key"){
3844 secret = value;
3845 }else if(key == "aws_session_token"){
3846 session_token = value;
3847 }
3848 }
3849
3850 if(profile != aws_profile){
3851 return EXIT_FAILURE;
3852 }
3853 if (session_token.empty()) {
3854 if (is_use_session_token) {
3855 S3FS_PRN_EXIT("AWS session token was expected but wasn't provided in aws/credentials file for profile: %s.", aws_profile.c_str());
3856 return EXIT_FAILURE;
3857 }
3858 if(!S3fsCurl::SetAccessKey(accesskey.c_str(), secret.c_str())){
3859 S3FS_PRN_EXIT("failed to set internal data for access key/secret key from aws credential file.");
3860 return EXIT_FAILURE;
3861 }
3862 } else {
3863 if (!S3fsCurl::SetAccessKeyWithSessionToken(accesskey.c_str(), secret.c_str(), session_token.c_str())) {
3864 S3FS_PRN_EXIT("session token is invalid.");
3865 return EXIT_FAILURE;
3866 }
3867 }
3868
3869 return EXIT_SUCCESS;
3870 }
3871
3872 //
3873 // read_passwd_file
3874 //
3875 // Support for per bucket credentials
3876 //
3877 // Format for the credentials file:
3878 // [bucket:]AccessKeyId:SecretAccessKey
3879 //
3880 // Lines beginning with # are considered comments
3881 // and ignored, as are empty lines
3882 //
3883 // Uncommented lines without the ":" character are flagged as
3884 // an error, so are lines with spaces or tabs
3885 //
3886 // only one default key pair is allowed, but not required
3887 //
3888 static int read_passwd_file()
3889 {
3890 bucketkvmap_t bucketmap;
3891 kvmap_t keyval;
3892 int result;
3893
3894 // if you got here, the password file
3895 // exists and is readable by the
3896 // current user, check for permissions
3897 if(EXIT_SUCCESS != check_passwd_file_perms()){
3898 return EXIT_FAILURE;
3899 }
3900
3901 //
3902 // parse passwd file
3903 //
3904 result = parse_passwd_file(bucketmap);
3905 if(-1 == result){
3906 return EXIT_FAILURE;
3907 }
3908
3909 //
3910 // check key=value type format.
3911 //
3912 bucketkvmap_t::iterator it = bucketmap.find(KEYVAL_FIELDS_TYPE);
3913 if(bucketmap.end() != it){
3914 // aws format
3915 result = check_for_aws_format(it->second);
3916 if(-1 == result){
3917 return EXIT_FAILURE;
3918 }else if(1 == result){
3919 // success to set
3920 return EXIT_SUCCESS;
3921 }
3922 }
3923
3924 std::string bucket_key = ALLBUCKET_FIELDS_TYPE;
3925 if(!bucket.empty() && bucketmap.end() != bucketmap.find(bucket)){
3926 bucket_key = bucket;
3927 }
3928 it = bucketmap.find(bucket_key);
3929 if(bucketmap.end() == it){
3930 S3FS_PRN_EXIT("Not found access key/secret key in passwd file.");
3931 return EXIT_FAILURE;
3932 }
3933 keyval = it->second;
3934 kvmap_t::iterator aws_accesskeyid_it = keyval.find(AWS_ACCESSKEYID);
3935 kvmap_t::iterator aws_secretkey_it = keyval.find(AWS_SECRETKEY);
3936 if(keyval.end() == aws_accesskeyid_it || keyval.end() == aws_secretkey_it){
3937 S3FS_PRN_EXIT("Not found access key/secret key in passwd file.");
3938 return EXIT_FAILURE;
3939 }
3940 if(!S3fsCurl::SetAccessKey(aws_accesskeyid_it->second.c_str(), aws_secretkey_it->second.c_str())){
3941 S3FS_PRN_EXIT("failed to set internal data for access key/secret key from passwd file.");
3942 return EXIT_FAILURE;
3943 }
3944 return EXIT_SUCCESS;
3945 }
3946
3947 //
3948 // get_access_keys
3949 //
3950 // called only when were are not mounting a
3951 // public bucket
3952 //
3953 // Here is the order precedence for getting the
3954 // keys:
3955 //
3956 // 1 - from the command line (security risk)
3957 // 2 - from a password file specified on the command line
3958 // 3 - from environment variables
3959 // 3a - from the AWS_CREDENTIAL_FILE environment variable
3960 // 3b - from ${HOME}/.aws/credentials
3961 // 4 - from the users ~/.passwd-s3fs
3962 // 5 - from /etc/passwd-s3fs
3963 //
3964 static int get_access_keys()
3965 {
3966 // should be redundant
3967 if(S3fsCurl::IsPublicBucket()){
3968 return EXIT_SUCCESS;
3969 }
3970
3971 // access key loading is deferred
3972 if(load_iamrole || is_ecs){
3973 return EXIT_SUCCESS;
3974 }
3975
3976 // 1 - keys specified on the command line
3977 if(S3fsCurl::IsSetAccessKeys()){
3978 return EXIT_SUCCESS;
3979 }
3980
3981 // 2 - was specified on the command line
3982 if(!passwd_file.empty()){
3983 std::ifstream PF(passwd_file.c_str());
3984 if(PF.good()){
3985 PF.close();
3986 return read_passwd_file();
3987 }else{
3988 S3FS_PRN_EXIT("specified passwd_file is not readable.");
3989 return EXIT_FAILURE;
3990 }
3991 }
3992
3993 // 3 - environment variables
3994 char* AWSACCESSKEYID = getenv("AWS_ACCESS_KEY_ID") ? getenv("AWS_ACCESS_KEY_ID") : getenv("AWSACCESSKEYID");
3995 char* AWSSECRETACCESSKEY = getenv("AWS_SECRET_ACCESS_KEY") ? getenv("AWS_SECRET_ACCESS_KEY") : getenv("AWSSECRETACCESSKEY");
3996 char* AWSSESSIONTOKEN = getenv("AWS_SESSION_TOKEN") ? getenv("AWS_SESSION_TOKEN") : getenv("AWSSESSIONTOKEN");
3997
3998 if(AWSACCESSKEYID != NULL || AWSSECRETACCESSKEY != NULL){
3999 if( (AWSACCESSKEYID == NULL && AWSSECRETACCESSKEY != NULL) ||
4000 (AWSACCESSKEYID != NULL && AWSSECRETACCESSKEY == NULL) ){
4001 S3FS_PRN_EXIT("both environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set together.");
4002 return EXIT_FAILURE;
4003 }
4004 S3FS_PRN_INFO2("access key from env variables");
4005 if (AWSSESSIONTOKEN != NULL) {
4006 S3FS_PRN_INFO2("session token is available");
4007 if (!S3fsCurl::SetAccessKeyWithSessionToken(AWSACCESSKEYID, AWSSECRETACCESSKEY, AWSSESSIONTOKEN)) {
4008 S3FS_PRN_EXIT("session token is invalid.");
4009 return EXIT_FAILURE;
4010 }
4011 } else {
4012 S3FS_PRN_INFO2("session token is not available");
4013 if (is_use_session_token) {
4014 S3FS_PRN_EXIT("environment variable AWS_SESSION_TOKEN is expected to be set.");
4015 return EXIT_FAILURE;
4016 }
4017 }
4018 if(!S3fsCurl::SetAccessKey(AWSACCESSKEYID, AWSSECRETACCESSKEY)){
4019 S3FS_PRN_EXIT("if one access key is specified, both keys need to be specified.");
4020 return EXIT_FAILURE;
4021 }
4022 return EXIT_SUCCESS;
4023 }
4024
4025 // 3a - from the AWS_CREDENTIAL_FILE environment variable
4026 char * AWS_CREDENTIAL_FILE;
4027 AWS_CREDENTIAL_FILE = getenv("AWS_CREDENTIAL_FILE");
4028 if(AWS_CREDENTIAL_FILE != NULL){
4029 passwd_file = AWS_CREDENTIAL_FILE;
4030 if(!passwd_file.empty()){
4031 std::ifstream PF(passwd_file.c_str());
4032 if(PF.good()){
4033 PF.close();
4034 return read_passwd_file();
4035 }else{
4036 S3FS_PRN_EXIT("AWS_CREDENTIAL_FILE: \"%s\" is not readable.", passwd_file.c_str());
4037 return EXIT_FAILURE;
4038 }
4039 }
4040 }
4041
4042 // 3b - check ${HOME}/.aws/credentials
4043 std::string aws_credentials = std::string(getpwuid(getuid())->pw_dir) + "/.aws/credentials";
4044 if(read_aws_credentials_file(aws_credentials) == EXIT_SUCCESS) {
4045 return EXIT_SUCCESS;
4046 }else if(aws_profile != "default"){
4047 S3FS_PRN_EXIT("Could not find profile: %s in file: %s", aws_profile.c_str(), aws_credentials.c_str());
4048 return EXIT_FAILURE;
4049 }
4050
4051 // 4 - from the default location in the users home directory
4052 char * HOME;
4053 HOME = getenv ("HOME");
4054 if(HOME != NULL){
4055 passwd_file = HOME;
4056 passwd_file += "/.passwd-s3fs";
4057 std::ifstream PF(passwd_file.c_str());
4058 if(PF.good()){
4059 PF.close();
4060 if(EXIT_SUCCESS != read_passwd_file()){
4061 return EXIT_FAILURE;
4062 }
4063 // It is possible that the user's file was there but
4064 // contained no key pairs i.e. commented out
4065 // in that case, go look in the final location
4066 if(S3fsCurl::IsSetAccessKeys()){
4067 return EXIT_SUCCESS;
4068 }
4069 }
4070 }
4071
4072 // 5 - from the system default location
4073 passwd_file = "/etc/passwd-s3fs";
4074 std::ifstream PF(passwd_file.c_str());
4075 if(PF.good()){
4076 PF.close();
4077 return read_passwd_file();
4078 }
4079 S3FS_PRN_EXIT("could not determine how to establish security credentials.");
4080
4081 return EXIT_FAILURE;
4082 }
4083
4084 //
4085 // Check & Set attributes for mount point.
4086 //
4087 static bool set_mountpoint_attribute(struct stat& mpst)
4088 {
4089 mp_uid = geteuid();
4090 mp_gid = getegid();
4091 mp_mode = S_IFDIR | (allow_other ? (is_mp_umask ? (~mp_umask & (S_IRWXU | S_IRWXG | S_IRWXO)) : (S_IRWXU | S_IRWXG | S_IRWXO)) : S_IRWXU);
4092
4093 S3FS_PRN_INFO2("PROC(uid=%u, gid=%u) - MountPoint(uid=%u, gid=%u, mode=%04o)",
4094 (unsigned int)mp_uid, (unsigned int)mp_gid, (unsigned int)(mpst.st_uid), (unsigned int)(mpst.st_gid), mpst.st_mode);
4095
4096 // check owner
4097 if(0 == mp_uid || mpst.st_uid == mp_uid){
4098 return true;
4099 }
4100 // check group permission
4101 if(mpst.st_gid == mp_gid || 1 == is_uid_include_group(mp_uid, mpst.st_gid)){
4102 if(S_IRWXG == (mpst.st_mode & S_IRWXG)){
4103 return true;
4104 }
4105 }
4106 // check other permission
4107 if(S_IRWXO == (mpst.st_mode & S_IRWXO)){
4108 return true;
4109 }
4110 return false;
4111 }
4112
4113 //
4114 // Set bucket and mount_prefix based on passed bucket name.
4115 //
4116 static int set_bucket(const char* arg)
4117 {
4118 char *bucket_name = (char*)arg;
4119 if(strstr(arg, ":")){
4120 if(strstr(arg, "://")){
4121 S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg);
4122 return -1;
4123 }
4124 bucket = strtok(bucket_name, ":");
4125 char* pmount_prefix = strtok(NULL, "");
4126 if(pmount_prefix){
4127 if(0 == strlen(pmount_prefix) || '/' != pmount_prefix[0]){
4128 S3FS_PRN_EXIT("path(%s) must be prefix \"/\".", pmount_prefix);
4129 return -1;
4130 }
4131 mount_prefix = pmount_prefix;
4132 // remove trailing slash
4133 if(mount_prefix[mount_prefix.size() - 1] == '/'){
4134 mount_prefix.erase(mount_prefix.size() - 1);
4135 }
4136 }
4137 }else{
4138 bucket = arg;
4139 }
4140 return 0;
4141 }
4142
4143 // This is repeatedly called by the fuse option parser
4144 // if the key is equal to FUSE_OPT_KEY_OPT, it's an option passed in prefixed by
4145 // '-' or '--' e.g.: -f -d -ousecache=/tmp
4146 //
4147 // if the key is equal to FUSE_OPT_KEY_NONOPT, it's either the bucket name
4148 // or the mountpoint. The bucket name will always come before the mountpoint
4149 static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_args* outargs)
4150 {
4151 int ret;
4152 if(key == FUSE_OPT_KEY_NONOPT){
4153 // the first NONOPT option is the bucket name
4154 if(bucket.empty()){
4155 if ((ret = set_bucket(arg))){
4156 return ret;
4157 }
4158 return 0;
4159 }else if (!strcmp(arg, "s3fs")) {
4160 return 0;
4161 }
4162
4163 // the second NONOPT option is the mountpoint(not utility mode)
4164 if(mountpoint.empty() && NO_UTILITY_MODE == utility_mode){
4165 // save the mountpoint and do some basic error checking
4166 mountpoint = arg;
4167 struct stat stbuf;
4168
4169 if(stat(arg, &stbuf) == -1){
4170 S3FS_PRN_EXIT("unable to access MOUNTPOINT %s: %s", mountpoint.c_str(), strerror(errno));
4171 return -1;
4172 }
4173 if(!(S_ISDIR(stbuf.st_mode))){
4174 S3FS_PRN_EXIT("MOUNTPOINT: %s is not a directory.", mountpoint.c_str());
4175 return -1;
4176 }
4177 if(!set_mountpoint_attribute(stbuf)){
4178 S3FS_PRN_EXIT("MOUNTPOINT: %s permission denied.", mountpoint.c_str());
4179 return -1;
4180 }
4181
4182 if(!nonempty){
4183 struct dirent *ent;
4184 DIR *dp = opendir(mountpoint.c_str());
4185 if(dp == NULL){
4186 S3FS_PRN_EXIT("failed to open MOUNTPOINT: %s: %s", mountpoint.c_str(), strerror(errno));
4187 return -1;
4188 }
4189 while((ent = readdir(dp)) != NULL){
4190 if(strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0){
4191 closedir(dp);
4192 S3FS_PRN_EXIT("MOUNTPOINT directory %s is not empty. if you are sure this is safe, can use the 'nonempty' mount option.", mountpoint.c_str());
4193 return -1;
4194 }
4195 }
4196 closedir(dp);
4197 }
4198 return 1;
4199 }
4200
4201 // Unknown option
4202 if(NO_UTILITY_MODE == utility_mode){
4203 S3FS_PRN_EXIT("specified unknown third option(%s).", arg);
4204 }else{
4205 S3FS_PRN_EXIT("specified unknown second option(%s). you don't need to specify second option(mountpoint) for utility mode(-u).", arg);
4206 }
4207 return -1;
4208
4209 }else if(key == FUSE_OPT_KEY_OPT){
4210 if(is_prefix(arg, "uid=")){
4211 s3fs_uid = get_uid(strchr(arg, '=') + sizeof(char));
4212 if(0 != geteuid() && 0 == s3fs_uid){
4213 S3FS_PRN_EXIT("root user can only specify uid=0.");
4214 return -1;
4215 }
4216 is_s3fs_uid = true;
4217 return 1; // continue for fuse option
4218 }
4219 if(is_prefix(arg, "gid=")){
4220 s3fs_gid = get_gid(strchr(arg, '=') + sizeof(char));
4221 if(0 != getegid() && 0 == s3fs_gid){
4222 S3FS_PRN_EXIT("root user can only specify gid=0.");
4223 return -1;
4224 }
4225 is_s3fs_gid = true;
4226 return 1; // continue for fuse option
4227 }
4228 if(is_prefix(arg, "umask=")){
4229 off_t s3fs_umask_tmp = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 8);
4230 s3fs_umask = s3fs_umask_tmp & (S_IRWXU | S_IRWXG | S_IRWXO);
4231 is_s3fs_umask = true;
4232 return 1; // continue for fuse option
4233 }
4234 if(0 == strcmp(arg, "allow_other")){
4235 allow_other = true;
4236 return 1; // continue for fuse option
4237 }
4238 if(is_prefix(arg, "mp_umask=")){
4239 off_t mp_umask_tmp = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 8);
4240 mp_umask = mp_umask_tmp & (S_IRWXU | S_IRWXG | S_IRWXO);
4241 is_mp_umask = true;
4242 return 0;
4243 }
4244 if(is_prefix(arg, "default_acl=")){
4245 const char* acl_string = strchr(arg, '=') + sizeof(char);
4246 acl_t acl = acl_t::from_str(acl_string);
4247 if(acl == acl_t::UNKNOWN){
4248 S3FS_PRN_EXIT("unknown value for default_acl: %s", acl_string);
4249 return -1;
4250 }
4251 S3fsCurl::SetDefaultAcl(acl);
4252 return 0;
4253 }
4254 if(is_prefix(arg, "retries=")){
4255 off_t retries = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10);
4256 if(retries == 0){
4257 S3FS_PRN_EXIT("retries must be greater than zero");
4258 return -1;
4259 }
4260 S3fsCurl::SetRetries(static_cast<int>(retries));
4261 return 0;
4262 }
4263 if(is_prefix(arg, "tmpdir=")){
4264 FdManager::SetTmpDir(strchr(arg, '=') + sizeof(char));
4265 return 0;
4266 }
4267 if(is_prefix(arg, "use_cache=")){
4268 FdManager::SetCacheDir(strchr(arg, '=') + sizeof(char));
4269 return 0;
4270 }
4271 if(0 == strcmp(arg, "check_cache_dir_exist")){
4272 FdManager::SetCheckCacheDirExist(true);
4273 return 0;
4274 }
4275 if(0 == strcmp(arg, "del_cache")){
4276 is_remove_cache = true;
4277 return 0;
4278 }
4279 if(is_prefix(arg, "multireq_max=")){
4280 int maxreq = static_cast<int>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4281 S3fsCurl::SetMaxMultiRequest(maxreq);
4282 return 0;
4283 }
4284 if(0 == strcmp(arg, "nonempty")){
4285 nonempty = true;
4286 return 1; // need to continue for fuse.
4287 }
4288 if(0 == strcmp(arg, "nomultipart")){
4289 nomultipart = true;
4290 return 0;
4291 }
4292 // old format for storage_class
4293 if(0 == strcmp(arg, "use_rrs") || is_prefix(arg, "use_rrs=")){
4294 off_t rrs = 1;
4295 // for an old format.
4296 if(is_prefix(arg, "use_rrs=")){
4297 rrs = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10);
4298 }
4299 if(0 == rrs){
4300 S3fsCurl::SetStorageClass("STANDARD");
4301 }else if(1 == rrs){
4302 S3fsCurl::SetStorageClass("REDUCED_REDUNDANCY");
4303 }else{
4304 S3FS_PRN_EXIT("poorly formed argument to option: use_rrs");
4305 return -1;
4306 }
4307 return 0;
4308 }
4309 if(is_prefix(arg, "storage_class=")){
4310 const char *storage_class = strchr(arg, '=') + sizeof(char);
4311 S3fsCurl::SetStorageClass(storage_class);
4312 return 0;
4313 }
4314 //
4315 // [NOTE]
4316 // use_sse Set Server Side Encrypting type to SSE-S3
4317 // use_sse=1
4318 // use_sse=file Set Server Side Encrypting type to Custom key(SSE-C) and load custom keys
4319 // use_sse=custom(c):file
4320 // use_sse=custom(c) Set Server Side Encrypting type to Custom key(SSE-C)
4321 // use_sse=kmsid(k):kms-key-id Set Server Side Encrypting type to AWS Key Management key id(SSE-KMS) and load KMS id
4322 // use_sse=kmsid(k) Set Server Side Encrypting type to AWS Key Management key id(SSE-KMS)
4323 //
4324 // load_sse_c=file Load Server Side Encrypting custom keys
4325 //
4326 // AWSSSECKEYS Loading Environment for Server Side Encrypting custom keys
4327 // AWSSSEKMSID Loading Environment for Server Side Encrypting Key id
4328 //
4329 if(is_prefix(arg, "use_sse")){
4330 if(0 == strcmp(arg, "use_sse") || 0 == strcmp(arg, "use_sse=1")){ // use_sse=1 is old type parameter
4331 // sse type is SSE_S3
4332 if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseS3Type()){
4333 S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment.");
4334 return -1;
4335 }
4336 S3fsCurl::SetSseType(sse_type_t::SSE_S3);
4337
4338 }else if(0 == strcmp(arg, "use_sse=kmsid") || 0 == strcmp(arg, "use_sse=k")){
4339 // sse type is SSE_KMS with out kmsid(expecting id is loaded by environment)
4340 if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseKmsType()){
4341 S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment.");
4342 return -1;
4343 }
4344 if(!S3fsCurl::IsSetSseKmsId()){
4345 S3FS_PRN_EXIT("use_sse=kms but not loaded kms id by environment.");
4346 return -1;
4347 }
4348 S3fsCurl::SetSseType(sse_type_t::SSE_KMS);
4349
4350 }else if(is_prefix(arg, "use_sse=kmsid:") || is_prefix(arg, "use_sse=k:")){
4351 // sse type is SSE_KMS with kmsid
4352 if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseKmsType()){
4353 S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment.");
4354 return -1;
4355 }
4356 const char* kmsid;
4357 if(is_prefix(arg, "use_sse=kmsid:")){
4358 kmsid = &arg[strlen("use_sse=kmsid:")];
4359 }else{
4360 kmsid = &arg[strlen("use_sse=k:")];
4361 }
4362 if(!S3fsCurl::SetSseKmsid(kmsid)){
4363 S3FS_PRN_EXIT("failed to load use_sse kms id.");
4364 return -1;
4365 }
4366 S3fsCurl::SetSseType(sse_type_t::SSE_KMS);
4367
4368 }else if(0 == strcmp(arg, "use_sse=custom") || 0 == strcmp(arg, "use_sse=c")){
4369 // sse type is SSE_C with out custom keys(expecting keys are loaded by environment or load_sse_c option)
4370 if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseCType()){
4371 S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment.");
4372 return -1;
4373 }
4374 // [NOTE]
4375 // do not check ckeys exists here.
4376 //
4377 S3fsCurl::SetSseType(sse_type_t::SSE_C);
4378
4379 }else if(is_prefix(arg, "use_sse=custom:") || is_prefix(arg, "use_sse=c:")){
4380 // sse type is SSE_C with custom keys
4381 if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseCType()){
4382 S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment.");
4383 return -1;
4384 }
4385 const char* ssecfile;
4386 if(is_prefix(arg, "use_sse=custom:")){
4387 ssecfile = &arg[strlen("use_sse=custom:")];
4388 }else{
4389 ssecfile = &arg[strlen("use_sse=c:")];
4390 }
4391 if(!S3fsCurl::SetSseCKeys(ssecfile)){
4392 S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile);
4393 return -1;
4394 }
4395 S3fsCurl::SetSseType(sse_type_t::SSE_C);
4396
4397 }else if(0 == strcmp(arg, "use_sse=")){ // this type is old style(parameter is custom key file path)
4398 // SSE_C with custom keys.
4399 const char* ssecfile = &arg[strlen("use_sse=")];
4400 if(!S3fsCurl::SetSseCKeys(ssecfile)){
4401 S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile);
4402 return -1;
4403 }
4404 S3fsCurl::SetSseType(sse_type_t::SSE_C);
4405
4406 }else{
4407 // never come here.
4408 S3FS_PRN_EXIT("something wrong use_sse option.");
4409 return -1;
4410 }
4411 return 0;
4412 }
4413 // [NOTE]
4414 // Do only load SSE custom keys, care for set without set sse type.
4415 if(is_prefix(arg, "load_sse_c=")){
4416 const char* ssecfile = &arg[strlen("load_sse_c=")];
4417 if(!S3fsCurl::SetSseCKeys(ssecfile)){
4418 S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile);
4419 return -1;
4420 }
4421 return 0;
4422 }
4423 if(is_prefix(arg, "ssl_verify_hostname=")){
4424 long sslvh = static_cast<long>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4425 if(-1 == S3fsCurl::SetSslVerifyHostname(sslvh)){
4426 S3FS_PRN_EXIT("poorly formed argument to option: ssl_verify_hostname.");
4427 return -1;
4428 }
4429 return 0;
4430 }
4431 if(is_prefix(arg, "passwd_file=")){
4432 passwd_file = strchr(arg, '=') + sizeof(char);
4433 return 0;
4434 }
4435 if(0 == strcmp(arg, "ibm_iam_auth")){
4436 S3fsCurl::SetIsIBMIAMAuth(true);
4437 S3fsCurl::SetIAMCredentialsURL("https://iam.cloud.ibm.com/identity/token");
4438 S3fsCurl::SetIAMTokenField("\"access_token\"");
4439 S3fsCurl::SetIAMExpiryField("\"expiration\"");
4440 S3fsCurl::SetIAMFieldCount(2);
4441 S3fsCurl::SetIMDSVersion(1);
4442 is_ibm_iam_auth = true;
4443 return 0;
4444 }
4445 if (0 == strcmp(arg, "use_session_token")) {
4446 is_use_session_token = true;
4447 return 0;
4448 }
4449 if(is_prefix(arg, "ibm_iam_endpoint=")){
4450 std::string endpoint_url;
4451 const char *iam_endpoint = strchr(arg, '=') + sizeof(char);
4452 // Check url for http / https protocol std::string
4453 if(!is_prefix(iam_endpoint, "https://") && !is_prefix(iam_endpoint, "http://")) {
4454 S3FS_PRN_EXIT("option ibm_iam_endpoint has invalid format, missing http / https protocol");
4455 return -1;
4456 }
4457 endpoint_url = std::string(iam_endpoint) + "/identity/token";
4458 S3fsCurl::SetIAMCredentialsURL(endpoint_url.c_str());
4459 return 0;
4460 }
4461 if(0 == strcmp(arg, "imdsv1only")){
4462 S3fsCurl::SetIMDSVersion(1);
4463 return 0;
4464 }
4465 if(0 == strcmp(arg, "ecs")){
4466 if (is_ibm_iam_auth) {
4467 S3FS_PRN_EXIT("option ecs cannot be used in conjunction with ibm");
4468 return -1;
4469 }
4470 S3fsCurl::SetIsECS(true);
4471 S3fsCurl::SetIMDSVersion(1);
4472 S3fsCurl::SetIAMCredentialsURL("http://169.254.170.2");
4473 S3fsCurl::SetIAMFieldCount(5);
4474 is_ecs = true;
4475 return 0;
4476 }
4477 if(is_prefix(arg, "iam_role")){
4478 if (is_ecs || is_ibm_iam_auth) {
4479 S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs or ibm");
4480 return -1;
4481 }
4482 if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
4483 // loading IAM role name in s3fs_init(), because we need to wait initializing curl.
4484 //
4485 load_iamrole = true;
4486 return 0;
4487
4488 }else if(is_prefix(arg, "iam_role=")){
4489 const char* role = strchr(arg, '=') + sizeof(char);
4490 S3fsCurl::SetIAMRole(role);
4491 load_iamrole = false;
4492 return 0;
4493 }
4494 }
4495 if(is_prefix(arg, "profile=")){
4496 aws_profile = strchr(arg, '=') + sizeof(char);
4497 return 0;
4498 }
4499 if(is_prefix(arg, "public_bucket=")){
4500 off_t pubbucket = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10);
4501 if(1 == pubbucket){
4502 S3fsCurl::SetPublicBucket(true);
4503 // [NOTE]
4504 // if bucket is public(without credential), s3 do not allow copy api.
4505 // so s3fs sets nocopyapi mode.
4506 //
4507 nocopyapi = true;
4508 }else if(0 == pubbucket){
4509 S3fsCurl::SetPublicBucket(false);
4510 }else{
4511 S3FS_PRN_EXIT("poorly formed argument to option: public_bucket.");
4512 return -1;
4513 }
4514 return 0;
4515 }
4516 if(is_prefix(arg, "bucket=")){
4517 std::string bname = strchr(arg, '=') + sizeof(char);
4518 if ((ret = set_bucket(bname.c_str()))){
4519 return ret;
4520 }
4521 return 0;
4522 }
4523 if(0 == strcmp(arg, "no_check_certificate")){
4524 S3fsCurl::SetCheckCertificate(false);
4525 return 0;
4526 }
4527 if(is_prefix(arg, "connect_timeout=")){
4528 long contimeout = static_cast<long>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4529 S3fsCurl::SetConnectTimeout(contimeout);
4530 return 0;
4531 }
4532 if(is_prefix(arg, "readwrite_timeout=")){
4533 time_t rwtimeout = static_cast<time_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4534 S3fsCurl::SetReadwriteTimeout(rwtimeout);
4535 return 0;
4536 }
4537 if(is_prefix(arg, "list_object_max_keys=")){
4538 int max_keys = static_cast<int>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4539 if(max_keys < 1000){
4540 S3FS_PRN_EXIT("argument should be over 1000: list_object_max_keys");
4541 return -1;
4542 }
4543 max_keys_list_object = max_keys;
4544 return 0;
4545 }
4546 if(is_prefix(arg, "max_stat_cache_size=")){
4547 unsigned long cache_size = static_cast<unsigned long>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), 10));
4548 StatCache::getStatCacheData()->SetCacheSize(cache_size);
4549 return 0;
4550 }
4551 if(is_prefix(arg, "stat_cache_expire=")){
4552 time_t expr_time = static_cast<time_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), 10));
4553 StatCache::getStatCacheData()->SetExpireTime(expr_time);
4554 return 0;
4555 }
4556 // [NOTE]
4557 // This option is for compatibility old version.
4558 if(is_prefix(arg, "stat_cache_interval_expire=")){
4559 time_t expr_time = static_cast<time_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4560 StatCache::getStatCacheData()->SetExpireTime(expr_time, true);
4561 return 0;
4562 }
4563 if(0 == strcmp(arg, "enable_noobj_cache")){
4564 StatCache::getStatCacheData()->EnableCacheNoObject();
4565 return 0;
4566 }
4567 if(0 == strcmp(arg, "nodnscache")){
4568 S3fsCurl::SetDnsCache(false);
4569 return 0;
4570 }
4571 if(0 == strcmp(arg, "nosscache")){
4572 S3fsCurl::SetSslSessionCache(false);
4573 return 0;
4574 }
4575 if(is_prefix(arg, "parallel_count=") || is_prefix(arg, "parallel_upload=")){
4576 int maxpara = static_cast<int>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4577 if(0 >= maxpara){
4578 S3FS_PRN_EXIT("argument should be over 1: parallel_count");
4579 return -1;
4580 }
4581 S3fsCurl::SetMaxParallelCount(maxpara);
4582 return 0;
4583 }
4584 if(is_prefix(arg, "fd_page_size=")){
4585 S3FS_PRN_ERR("option fd_page_size is no longer supported, so skip this option.");
4586 return 0;
4587 }
4588 if(is_prefix(arg, "multipart_size=")){
4589 off_t size = static_cast<off_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4590 if(!S3fsCurl::SetMultipartSize(size)){
4591 S3FS_PRN_EXIT("multipart_size option must be at least 5 MB.");
4592 return -1;
4593 }
4594 return 0;
4595 }
4596 if(is_prefix(arg, "multipart_copy_size=")){
4597 off_t size = static_cast<off_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4598 if(!S3fsCurl::SetMultipartCopySize(size)){
4599 S3FS_PRN_EXIT("multipart_copy_size option must be at least 5 MB.");
4600 return -1;
4601 }
4602 return 0;
4603 }
4604 if(is_prefix(arg, "max_dirty_data=")){
4605 off_t size = static_cast<off_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
4606 if(size >= 50){
4607 size *= 1024 * 1024;
4608 }else if(size != -1){
4609 S3FS_PRN_EXIT("max_dirty_data option must be at least 50 MB.");
4610 return -1;
4611 }
4612 max_dirty_data = size;
4613 return 0;
4614 }
4615 if(is_prefix(arg, "ensure_diskfree=")){
4616 off_t dfsize = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10) * 1024 * 1024;
4617 if(dfsize < S3fsCurl::GetMultipartSize()){
4618 S3FS_PRN_WARN("specified size to ensure disk free space is smaller than multipart size, so set multipart size to it.");
4619 dfsize = S3fsCurl::GetMultipartSize();
4620 }
4621 FdManager::SetEnsureFreeDiskSpace(dfsize);
4622 return 0;
4623 }
4624 if(is_prefix(arg, "fake_diskfree=")){
4625 S3FS_PRN_WARN("The fake_diskfree option was specified. Use this option for testing or debugging.");
4626
4627 // [NOTE] This value is used for initializing to FdManager after parsing all options.
4628 fake_diskfree_size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10) * 1024 * 1024;
4629 return 0;
4630 }
4631 if(is_prefix(arg, "multipart_threshold=")){
4632 multipart_threshold = static_cast<int64_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)) * 1024 * 1024;
4633 if(multipart_threshold <= MIN_MULTIPART_SIZE){
4634 S3FS_PRN_EXIT("multipart_threshold must be at least %lld, was: %lld", static_cast<long long>(MIN_MULTIPART_SIZE), static_cast<long long>(multipart_threshold));
4635 return -1;
4636 }
4637 return 0;
4638 }
4639 if(is_prefix(arg, "singlepart_copy_limit=")){
4640 singlepart_copy_limit = static_cast<int64_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)) * 1024 * 1024;
4641 return 0;
4642 }
4643 if(is_prefix(arg, "ahbe_conf=")){
4644 std::string ahbe_conf = strchr(arg, '=') + sizeof(char);
4645 if(!AdditionalHeader::get()->Load(ahbe_conf.c_str())){
4646 S3FS_PRN_EXIT("failed to load ahbe_conf file(%s).", ahbe_conf.c_str());
4647 return -1;
4648 }
4649 AdditionalHeader::get()->Dump();
4650 return 0;
4651 }
4652 if(0 == strcmp(arg, "noxmlns")){
4653 noxmlns = true;
4654 return 0;
4655 }
4656 if(0 == strcmp(arg, "nomixupload")){
4657 FdEntity::SetNoMixMultipart();
4658 return 0;
4659 }
4660 if(0 == strcmp(arg, "nocopyapi")){
4661 nocopyapi = true;
4662 return 0;
4663 }
4664 if(0 == strcmp(arg, "norenameapi")){
4665 norenameapi = true;
4666 return 0;
4667 }
4668 if(0 == strcmp(arg, "complement_stat")){
4669 complement_stat = true;
4670 return 0;
4671 }
4672 if(0 == strcmp(arg, "notsup_compat_dir")){
4673 support_compat_dir = false;
4674 return 0;
4675 }
4676 if(0 == strcmp(arg, "enable_content_md5")){
4677 S3fsCurl::SetContentMd5(true);
4678 return 0;
4679 }
4680 if(is_prefix(arg, "host=")){
4681 s3host = strchr(arg, '=') + sizeof(char);
4682 return 0;
4683 }
4684 if(is_prefix(arg, "servicepath=")){
4685 service_path = strchr(arg, '=') + sizeof(char);
4686 return 0;
4687 }
4688 if(is_prefix(arg, "url=")){
4689 s3host = strchr(arg, '=') + sizeof(char);
4690 // strip the trailing '/', if any, off the end of the host
4691 // std::string
4692 size_t found, length;
4693 found = s3host.find_last_of('/');
4694 length = s3host.length();
4695 while(found == (length - 1) && length > 0){
4696 s3host.erase(found);
4697 found = s3host.find_last_of('/');
4698 length = s3host.length();
4699 }
4700 // Check url for http / https protocol std::string
4701 if(!is_prefix(s3host.c_str(), "https://") && !is_prefix(s3host.c_str(), "http://")){
4702 S3FS_PRN_EXIT("option url has invalid format, missing http / https protocol");
4703 return -1;
4704 }
4705 return 0;
4706 }
4707 if(0 == strcmp(arg, "sigv2")){
4708 S3fsCurl::SetSignatureType(V2_ONLY);
4709 return 0;
4710 }
4711 if(0 == strcmp(arg, "sigv4")){
4712 S3fsCurl::SetSignatureType(V4_ONLY);
4713 return 0;
4714 }
4715 if(0 == strcmp(arg, "createbucket")){
4716 create_bucket = true;
4717 return 0;
4718 }
4719 if(is_prefix(arg, "endpoint=")){
4720 endpoint = strchr(arg, '=') + sizeof(char);
4721 is_specified_endpoint = true;
4722 return 0;
4723 }
4724 if(0 == strcmp(arg, "use_path_request_style")){
4725 pathrequeststyle = true;
4726 return 0;
4727 }
4728 if(0 == strcmp(arg, "noua")){
4729 S3fsCurl::SetUserAgentFlag(false);
4730 return 0;
4731 }
4732 if(0 == strcmp(arg, "listobjectsv2")){
4733 S3fsCurl::SetListObjectsV2(true);
4734 return 0;
4735 }
4736 if(0 == strcmp(arg, "use_xattr")){
4737 is_use_xattr = true;
4738 return 0;
4739 }else if(is_prefix(arg, "use_xattr=")){
4740 const char* strflag = strchr(arg, '=') + sizeof(char);
4741 if(0 == strcmp(strflag, "1")){
4742 is_use_xattr = true;
4743 }else if(0 == strcmp(strflag, "0")){
4744 is_use_xattr = false;
4745 }else{
4746 S3FS_PRN_EXIT("option use_xattr has unknown parameter(%s).", strflag);
4747 return -1;
4748 }
4749 return 0;
4750 }
4751 if(is_prefix(arg, "cipher_suites=")){
4752 cipher_suites = strchr(arg, '=') + sizeof(char);
4753 return 0;
4754 }
4755 if(is_prefix(arg, "instance_name=")){
4756 instance_name = strchr(arg, '=') + sizeof(char);
4757 instance_name = "[" + instance_name + "]";
4758 return 0;
4759 }
4760 if(is_prefix(arg, "mime=")){
4761 mimetype_file = strchr(arg, '=') + sizeof(char);
4762 return 0;
4763 }
4764 //
4765 // log file option
4766 //
4767 if(is_prefix(arg, "logfile=")){
4768 const char* strlogfile = strchr(arg, '=') + sizeof(char);
4769 if(!S3fsLog::SetLogfile(strlogfile)){
4770 S3FS_PRN_EXIT("The file(%s) specified by logfile option could not be opened.", strlogfile);
4771 return -1;
4772 }
4773 return 0;
4774 }
4775 //
4776 // debug level option
4777 //
4778 if(is_prefix(arg, "dbglevel=")){
4779 const char* strlevel = strchr(arg, '=') + sizeof(char);
4780 if(0 == strcasecmp(strlevel, "silent") || 0 == strcasecmp(strlevel, "critical") || 0 == strcasecmp(strlevel, "crit")){
4781 S3fsLog::SetLogLevel(S3fsLog::LEVEL_CRIT);
4782 }else if(0 == strcasecmp(strlevel, "error") || 0 == strcasecmp(strlevel, "err")){
4783 S3fsLog::SetLogLevel(S3fsLog::LEVEL_ERR);
4784 }else if(0 == strcasecmp(strlevel, "wan") || 0 == strcasecmp(strlevel, "warn") || 0 == strcasecmp(strlevel, "warning")){
4785 S3fsLog::SetLogLevel(S3fsLog::LEVEL_WARN);
4786 }else if(0 == strcasecmp(strlevel, "inf") || 0 == strcasecmp(strlevel, "info") || 0 == strcasecmp(strlevel, "information")){
4787 S3fsLog::SetLogLevel(S3fsLog::LEVEL_INFO);
4788 }else if(0 == strcasecmp(strlevel, "dbg") || 0 == strcasecmp(strlevel, "debug")){
4789 S3fsLog::SetLogLevel(S3fsLog::LEVEL_DBG);
4790 }else{
4791 S3FS_PRN_EXIT("option dbglevel has unknown parameter(%s).", strlevel);
4792 return -1;
4793 }
4794 return 0;
4795 }
4796 //
4797 // debug option
4798 //
4799 // S3fsLog level is LEVEL_INFO, after second -d is passed to fuse.
4800 //
4801 if(0 == strcmp(arg, "-d") || 0 == strcmp(arg, "--debug")){
4802 if(!S3fsLog::IsS3fsLogInfo() && !S3fsLog::IsS3fsLogDbg()){
4803 S3fsLog::SetLogLevel(S3fsLog::LEVEL_INFO);
4804 return 0;
4805 }
4806 if(0 == strcmp(arg, "--debug")){
4807 // fuse doesn't understand "--debug", but it understands -d.
4808 // but we can't pass -d back to fuse.
4809 return 0;
4810 }
4811 }
4812 // "f2" is not used no more.
4813 // (set S3fsLog::LEVEL_DBG)
4814 if(0 == strcmp(arg, "f2")){
4815 S3fsLog::SetLogLevel(S3fsLog::LEVEL_DBG);
4816 return 0;
4817 }
4818 if(0 == strcmp(arg, "curldbg")){
4819 S3fsCurl::SetVerbose(true);
4820 return 0;
4821 }else if(is_prefix(arg, "curldbg=")){
4822 const char* strlevel = strchr(arg, '=') + sizeof(char);
4823 if(0 == strcasecmp(strlevel, "normal")){
4824 S3fsCurl::SetVerbose(true);
4825 }else if(0 == strcasecmp(strlevel, "body")){
4826 S3fsCurl::SetVerbose(true);
4827 S3fsCurl::SetDumpBody(true);
4828 }else{
4829 S3FS_PRN_EXIT("option curldbg has unknown parameter(%s).", strlevel);
4830 return -1;
4831 }
4832 return 0;
4833 }
4834 //
4835 // no time stamp in debug message
4836 //
4837 if(0 == strcmp(arg, "no_time_stamp_msg")){
4838 S3fsLog::SetTimeStamp(false);
4839 return 0;
4840 }
4841 //
4842 // Check cache file, using SIGUSR1
4843 //
4844 if(0 == strcmp(arg, "set_check_cache_sigusr1")){
4845 if(!S3fsSignals::SetUsr1Handler(NULL)){
4846 S3FS_PRN_EXIT("could not set sigusr1 for checking cache.");
4847 return -1;
4848 }
4849 return 0;
4850 }else if(is_prefix(arg, "set_check_cache_sigusr1=")){
4851 const char* strfilepath = strchr(arg, '=') + sizeof(char);
4852 if(!S3fsSignals::SetUsr1Handler(strfilepath)){
4853 S3FS_PRN_EXIT("could not set sigusr1 for checking cache and output file(%s).", strfilepath);
4854 return -1;
4855 }
4856 return 0;
4857 }
4858 if(is_prefix(arg, "accessKeyId=")){
4859 S3FS_PRN_EXIT("option accessKeyId is no longer supported.");
4860 return -1;
4861 }
4862 if(is_prefix(arg, "secretAccessKey=")){
4863 S3FS_PRN_EXIT("option secretAccessKey is no longer supported.");
4864 return -1;
4865 }
4866 if(0 == strcmp(arg, "use_wtf8")){
4867 use_wtf8 = true;
4868 return 0;
4869 }
4870 if(0 == strcmp(arg, "requester_pays")){
4871 S3fsCurl::SetRequesterPays(true);
4872 return 0;
4873 }
4874 // [NOTE]
4875 // following option will be discarding, because these are not for fuse.
4876 // (Referenced sshfs.c)
4877 //
4878 if(0 == strcmp(arg, "auto") ||
4879 0 == strcmp(arg, "noauto") ||
4880 0 == strcmp(arg, "user") ||
4881 0 == strcmp(arg, "nouser") ||
4882 0 == strcmp(arg, "users") ||
4883 0 == strcmp(arg, "_netdev"))
4884 {
4885 return 0;
4886 }
4887 }
4888 return 1;
4889 }
4890
4891 int main(int argc, char* argv[])
4892 {
4893 int ch;
4894 int fuse_res;
4895 int option_index = 0;
4896 struct fuse_operations s3fs_oper;
4897 time_t incomp_abort_time = (24 * 60 * 60);
4898 S3fsLog singletonLog;
4899
4900 static const struct option long_opts[] = {
4901 {"help", no_argument, NULL, 'h'},
4902 {"version", no_argument, 0, 0},
4903 {"debug", no_argument, NULL, 'd'},
4904 {"incomplete-mpu-list", no_argument, NULL, 'u'},
4905 {"incomplete-mpu-abort", optional_argument, NULL, 'a'}, // 'a' is only identifier and is not option.
4906 {NULL, 0, NULL, 0}
4907 };
4908
4909 // init xml2
4910 xmlInitParser();
4911 LIBXML_TEST_VERSION
4912
4913 init_sysconf_vars();
4914
4915 // get program name - emulate basename
4916 program_name = argv[0];
4917 size_t found = program_name.find_last_of('/');
4918 if(found != std::string::npos){
4919 program_name.replace(0, found+1, "");
4920 }
4921
4922 while((ch = getopt_long(argc, argv, "dho:fsu", long_opts, &option_index)) != -1){
4923 switch(ch){
4924 case 0:
4925 if(strcmp(long_opts[option_index].name, "version") == 0){
4926 show_version();
4927 exit(EXIT_SUCCESS);
4928 }
4929 break;
4930 case 'h':
4931 show_help();
4932 exit(EXIT_SUCCESS);
4933 case 'o':
4934 break;
4935 case 'd':
4936 break;
4937 case 'f':
4938 foreground = true;
4939 break;
4940 case 's':
4941 break;
4942 case 'u': // --incomplete-mpu-list
4943 if(NO_UTILITY_MODE != utility_mode){
4944 S3FS_PRN_EXIT("already utility mode option is specified.");
4945 exit(EXIT_FAILURE);
4946 }
4947 utility_mode = INCOMP_TYPE_LIST;
4948 break;
4949 case 'a': // --incomplete-mpu-abort
4950 if(NO_UTILITY_MODE != utility_mode){
4951 S3FS_PRN_EXIT("already utility mode option is specified.");
4952 exit(EXIT_FAILURE);
4953 }
4954 utility_mode = INCOMP_TYPE_ABORT;
4955
4956 // check expire argument
4957 if(NULL != optarg && 0 == strcasecmp(optarg, "all")){ // all is 0s
4958 incomp_abort_time = 0;
4959 }else if(NULL != optarg){
4960 if(!convert_unixtime_from_option_arg(optarg, incomp_abort_time)){
4961 S3FS_PRN_EXIT("--incomplete-mpu-abort option argument is wrong.");
4962 exit(EXIT_FAILURE);
4963 }
4964 }
4965 // if optarg is null, incomp_abort_time is 24H(default)
4966 break;
4967 default:
4968 exit(EXIT_FAILURE);
4969 }
4970 }
4971 // print launch message
4972 print_launch_message(argc, argv);
4973
4974 // Load SSE environment
4975 if(!S3fsCurl::LoadEnvSse()){
4976 S3FS_PRN_EXIT("something wrong about SSE environment.");
4977 exit(EXIT_FAILURE);
4978 }
4979
4980 // ssl init
4981 if(!s3fs_init_global_ssl()){
4982 S3FS_PRN_EXIT("could not initialize for ssl libraries.");
4983 exit(EXIT_FAILURE);
4984 }
4985
4986 // init curl (without mime types)
4987 //
4988 // [NOTE]
4989 // The curl initialization here does not load mime types.
4990 // The mime types file parameter are dynamic values according
4991 // to the user's environment, and are analyzed by the my_fuse_opt_proc
4992 // function.
4993 // The my_fuse_opt_proc function is executed after this curl
4994 // initialization. Because the curl method is used in the
4995 // my_fuse_opt_proc function, then it must be called here to
4996 // initialize. Fortunately, the processing using mime types
4997 // is only PUT/POST processing, and it is not used until the
4998 // call of my_fuse_opt_proc function is completed. Therefore,
4999 // the mime type is loaded just after calling the my_fuse_opt_proc
5000 // function.
5001 //
5002 if(!S3fsCurl::InitS3fsCurl()){
5003 S3FS_PRN_EXIT("Could not initiate curl library.");
5004 s3fs_destroy_global_ssl();
5005 exit(EXIT_FAILURE);
5006 }
5007
5008 // clear this structure
5009 memset(&s3fs_oper, 0, sizeof(s3fs_oper));
5010
5011 // This is the fuse-style parser for the arguments
5012 // after which the bucket name and mountpoint names
5013 // should have been set
5014 struct fuse_args custom_args = FUSE_ARGS_INIT(argc, argv);
5015 if(0 != fuse_opt_parse(&custom_args, NULL, NULL, my_fuse_opt_proc)){
5016 S3fsCurl::DestroyS3fsCurl();
5017 s3fs_destroy_global_ssl();
5018 exit(EXIT_FAILURE);
5019 }
5020
5021 // init mime types for curl
5022 if(!S3fsCurl::InitMimeType(mimetype_file)){
5023 S3FS_PRN_WARN("Missing MIME types prevents setting Content-Type on uploaded objects.");
5024 }
5025
5026 // [NOTE]
5027 // exclusive option check here.
5028 //
5029 if(strcasecmp(S3fsCurl::GetStorageClass().c_str(), "REDUCED_REDUNDANCY") == 0 && !S3fsCurl::IsSseDisable()){
5030 S3FS_PRN_EXIT("use_sse option could not be specified with storage class reduced_redundancy.");
5031 S3fsCurl::DestroyS3fsCurl();
5032 s3fs_destroy_global_ssl();
5033 exit(EXIT_FAILURE);
5034 }
5035 if(!S3fsCurl::FinalCheckSse()){
5036 S3FS_PRN_EXIT("something wrong about SSE options.");
5037 S3fsCurl::DestroyS3fsCurl();
5038 s3fs_destroy_global_ssl();
5039 exit(EXIT_FAILURE);
5040 }
5041
5042 if(!FdEntity::GetNoMixMultipart() && max_dirty_data != -1){
5043 S3FS_PRN_WARN("Setting max_dirty_data to -1 when nomixupload is enabled");
5044 max_dirty_data = -1;
5045 }
5046
5047 // The first plain argument is the bucket
5048 if(bucket.empty()){
5049 S3FS_PRN_EXIT("missing BUCKET argument.");
5050 show_usage();
5051 S3fsCurl::DestroyS3fsCurl();
5052 s3fs_destroy_global_ssl();
5053 exit(EXIT_FAILURE);
5054 }
5055
5056 // bucket names cannot contain upper case characters in virtual-hosted style
5057 if((!pathrequeststyle) && (lower(bucket) != bucket)){
5058 S3FS_PRN_EXIT("BUCKET %s, name not compatible with virtual-hosted style.", bucket.c_str());
5059 S3fsCurl::DestroyS3fsCurl();
5060 s3fs_destroy_global_ssl();
5061 exit(EXIT_FAILURE);
5062 }
5063
5064 // check bucket name for illegal characters
5065 found = bucket.find_first_of("/:\\;!@#$%^&*?|+=");
5066 if(found != std::string::npos){
5067 S3FS_PRN_EXIT("BUCKET %s -- bucket name contains an illegal character.", bucket.c_str());
5068 S3fsCurl::DestroyS3fsCurl();
5069 s3fs_destroy_global_ssl();
5070 exit(EXIT_FAILURE);
5071 }
5072
5073 if(!pathrequeststyle && is_prefix(s3host.c_str(), "https://") && bucket.find_first_of('.') != std::string::npos) {
5074 S3FS_PRN_EXIT("BUCKET %s -- cannot mount bucket with . while using HTTPS without use_path_request_style", bucket.c_str());
5075 S3fsCurl::DestroyS3fsCurl();
5076 s3fs_destroy_global_ssl();
5077 exit(EXIT_FAILURE);
5078 }
5079
5080 // The second plain argument is the mountpoint
5081 // if the option was given, we all ready checked for a
5082 // readable, non-empty directory, this checks determines
5083 // if the mountpoint option was ever supplied
5084 if(NO_UTILITY_MODE == utility_mode){
5085 if(mountpoint.empty()){
5086 S3FS_PRN_EXIT("missing MOUNTPOINT argument.");
5087 show_usage();
5088 S3fsCurl::DestroyS3fsCurl();
5089 s3fs_destroy_global_ssl();
5090 exit(EXIT_FAILURE);
5091 }
5092 }
5093
5094 // error checking of command line arguments for compatibility
5095 if(S3fsCurl::IsPublicBucket() && S3fsCurl::IsSetAccessKeys()){
5096 S3FS_PRN_EXIT("specifying both public_bucket and the access keys options is invalid.");
5097 S3fsCurl::DestroyS3fsCurl();
5098 s3fs_destroy_global_ssl();
5099 exit(EXIT_FAILURE);
5100 }
5101 if(!passwd_file.empty() && S3fsCurl::IsSetAccessKeys()){
5102 S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid.");
5103 S3fsCurl::DestroyS3fsCurl();
5104 s3fs_destroy_global_ssl();
5105 exit(EXIT_FAILURE);
5106 }
5107 if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs){
5108 if(EXIT_SUCCESS != get_access_keys()){
5109 S3fsCurl::DestroyS3fsCurl();
5110 s3fs_destroy_global_ssl();
5111 exit(EXIT_FAILURE);
5112 }
5113 if(!S3fsCurl::IsSetAccessKeys()){
5114 S3FS_PRN_EXIT("could not establish security credentials, check documentation.");
5115 S3fsCurl::DestroyS3fsCurl();
5116 s3fs_destroy_global_ssl();
5117 exit(EXIT_FAILURE);
5118 }
5119 // More error checking on the access key pair can be done
5120 // like checking for appropriate lengths and characters
5121 }
5122
5123 // check tmp dir permission
5124 if(!FdManager::CheckTmpDirExist()){
5125 S3FS_PRN_EXIT("temporary directory doesn't exists.");
5126 S3fsCurl::DestroyS3fsCurl();
5127 s3fs_destroy_global_ssl();
5128 exit(EXIT_FAILURE);
5129 }
5130
5131 // check cache dir permission
5132 if(!FdManager::CheckCacheDirExist() || !FdManager::CheckCacheTopDir() || !CacheFileStat::CheckCacheFileStatTopDir()){
5133 S3FS_PRN_EXIT("could not allow cache directory permission, check permission of cache directories.");
5134 S3fsCurl::DestroyS3fsCurl();
5135 s3fs_destroy_global_ssl();
5136 exit(EXIT_FAILURE);
5137 }
5138
5139 // set fake free disk space
5140 if(-1 != fake_diskfree_size){
5141 FdManager::InitFakeUsedDiskSize(fake_diskfree_size);
5142 }
5143
5144 // check IBM IAM requirements
5145 if(is_ibm_iam_auth){
5146 // check that default ACL is either public-read or private
5147 acl_t defaultACL = S3fsCurl::GetDefaultAcl();
5148 if(defaultACL != acl_t::PRIVATE && defaultACL != acl_t::PUBLIC_READ){
5149 S3FS_PRN_EXIT("can only use 'public-read' or 'private' ACL while using ibm_iam_auth");
5150 S3fsCurl::DestroyS3fsCurl();
5151 s3fs_destroy_global_ssl();
5152 exit(EXIT_FAILURE);
5153 }
5154
5155 if(create_bucket && !S3fsCurl::IsSetAccessKeyID()){
5156 S3FS_PRN_EXIT("missing service instance ID for bucket creation");
5157 S3fsCurl::DestroyS3fsCurl();
5158 s3fs_destroy_global_ssl();
5159 exit(EXIT_FAILURE);
5160 }
5161 }
5162
5163 // set user agent
5164 S3fsCurl::InitUserAgent();
5165
5166 // There's room for more command line error checking
5167
5168 // Check to see if the bucket name contains periods and https (SSL) is
5169 // being used. This is a known limitation:
5170 // https://docs.amazonwebservices.com/AmazonS3/latest/dev/
5171 // The Developers Guide suggests that either use HTTP of for us to write
5172 // our own certificate verification logic.
5173 // For now, this will be unsupported unless we get a request for it to
5174 // be supported. In that case, we have a couple of options:
5175 // - implement a command line option that bypasses the verify host
5176 // but doesn't bypass verifying the certificate
5177 // - write our own host verification (this might be complex)
5178 // See issue #128strncasecmp
5179 /*
5180 if(1 == S3fsCurl::GetSslVerifyHostname()){
5181 found = bucket.find_first_of('.');
5182 if(found != std::string::npos){
5183 found = s3host.find("https:");
5184 if(found != std::string::npos){
5185 S3FS_PRN_EXIT("Using https and a bucket name with periods is unsupported.");
5186 exit(1);
5187 }
5188 }
5189 }
5190 */
5191
5192 if(NO_UTILITY_MODE != utility_mode){
5193 int exitcode = s3fs_utility_processing(incomp_abort_time);
5194
5195 S3fsCurl::DestroyS3fsCurl();
5196 s3fs_destroy_global_ssl();
5197 exit(exitcode);
5198 }
5199
5200 // Check multipart / copy api for mix multipart uploading
5201 if(nomultipart || nocopyapi || norenameapi){
5202 FdEntity::SetNoMixMultipart();
5203 max_dirty_data = -1;
5204 }
5205
5206 // check free disk space
5207 if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){
5208 S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by s3fs.");
5209 S3fsCurl::DestroyS3fsCurl();
5210 s3fs_destroy_global_ssl();
5211 exit(EXIT_FAILURE);
5212 }
5213
5214 s3fs_oper.getattr = s3fs_getattr;
5215 s3fs_oper.readlink = s3fs_readlink;
5216 s3fs_oper.mknod = s3fs_mknod;
5217 s3fs_oper.mkdir = s3fs_mkdir;
5218 s3fs_oper.unlink = s3fs_unlink;
5219 s3fs_oper.rmdir = s3fs_rmdir;
5220 s3fs_oper.symlink = s3fs_symlink;
5221 s3fs_oper.rename = s3fs_rename;
5222 s3fs_oper.link = s3fs_link;
5223 if(!nocopyapi){
5224 s3fs_oper.chmod = s3fs_chmod;
5225 s3fs_oper.chown = s3fs_chown;
5226 s3fs_oper.utimens = s3fs_utimens;
5227 }else{
5228 s3fs_oper.chmod = s3fs_chmod_nocopy;
5229 s3fs_oper.chown = s3fs_chown_nocopy;
5230 s3fs_oper.utimens = s3fs_utimens_nocopy;
5231 }
5232 s3fs_oper.truncate = s3fs_truncate;
5233 s3fs_oper.open = s3fs_open;
5234 s3fs_oper.read = s3fs_read;
5235 s3fs_oper.write = s3fs_write;
5236 s3fs_oper.statfs = s3fs_statfs;
5237 s3fs_oper.flush = s3fs_flush;
5238 s3fs_oper.fsync = s3fs_fsync;
5239 s3fs_oper.release = s3fs_release;
5240 s3fs_oper.opendir = s3fs_opendir;
5241 s3fs_oper.readdir = s3fs_readdir;
5242 s3fs_oper.init = s3fs_init;
5243 s3fs_oper.destroy = s3fs_destroy;
5244 s3fs_oper.access = s3fs_access;
5245 s3fs_oper.create = s3fs_create;
5246 // extended attributes
5247 if(is_use_xattr){
5248 s3fs_oper.setxattr = s3fs_setxattr;
5249 s3fs_oper.getxattr = s3fs_getxattr;
5250 s3fs_oper.listxattr = s3fs_listxattr;
5251 s3fs_oper.removexattr = s3fs_removexattr;
5252 }
5253
5254 // now passing things off to fuse, fuse will finish evaluating the command line args
5255 fuse_res = fuse_main(custom_args.argc, custom_args.argv, &s3fs_oper, NULL);
5256 fuse_opt_free_args(&custom_args);
5257
5258 // Destroy curl
5259 if(!S3fsCurl::DestroyS3fsCurl()){
5260 S3FS_PRN_WARN("Could not release curl library.");
5261 }
5262 s3fs_destroy_global_ssl();
5263
5264 // cleanup xml2
5265 xmlCleanupParser();
5266 S3FS_MALLOCTRIM(0);
5267
5268 exit(fuse_res);
5269 }
5270
5271 /*
5272 * Local variables:
5273 * tab-width: 4
5274 * c-basic-offset: 4
5275 * End:
5276 * vim600: expandtab sw=4 ts=4 fdm=marker
5277 * vim<600: expandtab sw=4 ts=4
5278 */
5279