1 #ifndef _WIN32_WINNT
2 #define _WIN32_WINNT 0x500
3 #endif
4
5 #include "common.h"
6 #include "utils.h"
7 #include "obj-backend.h"
8
9 #ifndef WIN32
10 #include <sys/types.h>
11 #include <sys/stat.h>
12 #include <fcntl.h>
13 #endif
14
15 #ifdef WIN32
16 #include <windows.h>
17 #include <io.h>
18 #endif
19
20 #define DEBUG_FLAG SEAFILE_DEBUG_OTHER
21 #include "log.h"
22
23 typedef struct FsPriv {
24 char *obj_dir;
25 int dir_len;
26 } FsPriv;
27
28 static void
id_to_path(FsPriv * priv,const char * obj_id,char path[],const char * repo_id,int version)29 id_to_path (FsPriv *priv, const char *obj_id, char path[],
30 const char *repo_id, int version)
31 {
32 char *pos = path;
33 int n;
34
35 #if defined MIGRATION || defined SEAFILE_CLIENT
36 if (version > 0) {
37 n = snprintf (path, SEAF_PATH_MAX, "%s/%s/", priv->obj_dir, repo_id);
38 pos += n;
39 }
40 #else
41 n = snprintf (path, SEAF_PATH_MAX, "%s/%s/", priv->obj_dir, repo_id);
42 pos += n;
43 #endif
44
45 memcpy (pos, obj_id, 2);
46 pos[2] = '/';
47 pos += 3;
48
49 memcpy (pos, obj_id + 2, 41 - 2);
50 }
51
52 static int
obj_backend_fs_read(ObjBackend * bend,const char * repo_id,int version,const char * obj_id,void ** data,int * len)53 obj_backend_fs_read (ObjBackend *bend,
54 const char *repo_id,
55 int version,
56 const char *obj_id,
57 void **data,
58 int *len)
59 {
60 char path[SEAF_PATH_MAX];
61 gsize tmp_len;
62 GError *error = NULL;
63
64 id_to_path (bend->priv, obj_id, path, repo_id, version);
65
66 /* seaf_debug ("object path: %s\n", path); */
67
68 g_file_get_contents (path, (gchar**)data, &tmp_len, &error);
69 if (error) {
70 #ifdef MIGRATION
71 g_clear_error (&error);
72 id_to_path (bend->priv, obj_id, path, repo_id, 1);
73 g_file_get_contents (path, (gchar**)data, &tmp_len, &error);
74 if (error) {
75 seaf_debug ("[obj backend] Failed to read object %s: %s.\n",
76 obj_id, error->message);
77 g_clear_error (&error);
78 return -1;
79 }
80 #else
81 seaf_debug ("[obj backend] Failed to read object %s: %s.\n",
82 obj_id, error->message);
83 g_clear_error (&error);
84 return -1;
85 #endif
86 }
87
88 *len = (int)tmp_len;
89 return 0;
90 }
91
92 /*
93 * Flush operating system and disk caches for @fd.
94 */
95 static int
fsync_obj_contents(int fd)96 fsync_obj_contents (int fd)
97 {
98 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__) || defined(__OpenBSD__)
99 /* Some file systems may not support fsync().
100 * In this case, just skip the error.
101 */
102 if (fsync (fd) < 0) {
103 if (errno == EINVAL)
104 return 0;
105 else {
106 seaf_warning ("Failed to fsync: %s.\n", strerror(errno));
107 return -1;
108 }
109 }
110 return 0;
111 #endif
112
113 #ifdef __APPLE__
114 /* OS X: fcntl() is required to flush disk cache, fsync() only
115 * flushes operating system cache.
116 */
117 if (fcntl (fd, F_FULLFSYNC, NULL) < 0) {
118 seaf_warning ("Failed to fsync: %s.\n", strerror(errno));
119 return -1;
120 }
121 return 0;
122 #endif
123
124 #ifdef WIN32
125 HANDLE handle;
126
127 handle = (HANDLE)_get_osfhandle (fd);
128 if (handle == INVALID_HANDLE_VALUE) {
129 seaf_warning ("Failed to get handle from fd.\n");
130 return -1;
131 }
132
133 if (!FlushFileBuffers (handle)) {
134 seaf_warning ("FlushFileBuffer() failed: %lu.\n", GetLastError());
135 return -1;
136 }
137
138 return 0;
139 #endif
140 }
141
142 /*
143 * Rename file from @tmp_path to @obj_path.
144 * This also makes sure the changes to @obj_path's parent folder
145 * is flushed to disk.
146 */
147 static int
rename_and_sync(const char * tmp_path,const char * obj_path)148 rename_and_sync (const char *tmp_path, const char *obj_path)
149 {
150 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__) || defined(__OpenBSD__)
151 char *parent_dir;
152 int ret = 0;
153
154 if (rename (tmp_path, obj_path) < 0) {
155 seaf_warning ("Failed to rename from %s to %s: %s.\n",
156 tmp_path, obj_path, strerror(errno));
157 return -1;
158 }
159
160 parent_dir = g_path_get_dirname (obj_path);
161 int dir_fd = open (parent_dir, O_RDONLY);
162 if (dir_fd < 0) {
163 seaf_warning ("Failed to open dir %s: %s.\n", parent_dir, strerror(errno));
164 goto out;
165 }
166
167 /* Some file systems don't support fsyncing a directory. Just ignore the error.
168 */
169 if (fsync (dir_fd) < 0) {
170 if (errno != EINVAL) {
171 seaf_warning ("Failed to fsync dir %s: %s.\n",
172 parent_dir, strerror(errno));
173 ret = -1;
174 }
175 goto out;
176 }
177
178 out:
179 g_free (parent_dir);
180 if (dir_fd >= 0)
181 close (dir_fd);
182 return ret;
183 #endif
184
185 #ifdef __APPLE__
186 /*
187 * OS X garantees an existence of obj_path always exists,
188 * even when the system crashes.
189 */
190 if (rename (tmp_path, obj_path) < 0) {
191 seaf_warning ("Failed to rename from %s to %s: %s.\n",
192 tmp_path, obj_path, strerror(errno));
193 return -1;
194 }
195 return 0;
196 #endif
197
198 #ifdef WIN32
199 wchar_t *w_tmp_path = g_utf8_to_utf16 (tmp_path, -1, NULL, NULL, NULL);
200 wchar_t *w_obj_path = g_utf8_to_utf16 (obj_path, -1, NULL, NULL, NULL);
201 int ret = 0;
202
203 if (!MoveFileExW (w_tmp_path, w_obj_path,
204 MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) {
205 seaf_warning ("MoveFilExW failed: %lu.\n", GetLastError());
206 ret = -1;
207 goto out;
208 }
209
210 out:
211 g_free (w_tmp_path);
212 g_free (w_obj_path);
213 return ret;
214 #endif
215 }
216
217 static int
save_obj_contents(const char * path,const void * data,int len,gboolean need_sync)218 save_obj_contents (const char *path, const void *data, int len, gboolean need_sync)
219 {
220 char tmp_path[SEAF_PATH_MAX];
221 int fd;
222
223 snprintf (tmp_path, SEAF_PATH_MAX, "%s.XXXXXX", path);
224 fd = g_mkstemp (tmp_path);
225 if (fd < 0) {
226 seaf_warning ("[obj backend] Failed to open tmp file %s: %s.\n",
227 tmp_path, strerror(errno));
228 return -1;
229 }
230
231 if (writen (fd, data, len) < 0) {
232 seaf_warning ("[obj backend] Failed to write obj %s: %s.\n",
233 tmp_path, strerror(errno));
234 return -1;
235 }
236
237 if (need_sync && fsync_obj_contents (fd) < 0)
238 return -1;
239
240 /* Close may return error, especially in NFS. */
241 if (close (fd) < 0) {
242 seaf_warning ("[obj backend Failed close obj %s: %s.\n",
243 tmp_path, strerror(errno));
244 return -1;
245 }
246
247 if (need_sync) {
248 if (rename_and_sync (tmp_path, path) < 0)
249 return -1;
250 } else {
251 if (g_rename (tmp_path, path) < 0) {
252 seaf_warning ("[obj backend] Failed to rename %s: %s.\n",
253 path, strerror(errno));
254 return -1;
255 }
256 }
257
258 return 0;
259 }
260
261 static int
create_parent_path(const char * path)262 create_parent_path (const char *path)
263 {
264 char *dir = g_path_get_dirname (path);
265 if (!dir)
266 return -1;
267
268 if (g_file_test (dir, G_FILE_TEST_EXISTS)) {
269 g_free (dir);
270 return 0;
271 }
272
273 if (g_mkdir_with_parents (dir, 0777) < 0) {
274 seaf_warning ("Failed to create object parent path %s: %s.\n",
275 dir, strerror(errno));
276 g_free (dir);
277 return -1;
278 }
279
280 g_free (dir);
281 return 0;
282 }
283
284 static int
obj_backend_fs_write(ObjBackend * bend,const char * repo_id,int version,const char * obj_id,void * data,int len,gboolean need_sync)285 obj_backend_fs_write (ObjBackend *bend,
286 const char *repo_id,
287 int version,
288 const char *obj_id,
289 void *data,
290 int len,
291 gboolean need_sync)
292 {
293 char path[SEAF_PATH_MAX];
294
295 id_to_path (bend->priv, obj_id, path, repo_id, version);
296
297 /* GTimeVal s, e; */
298
299 /* g_get_current_time (&s); */
300
301 if (create_parent_path (path) < 0) {
302 seaf_warning ("[obj backend] Failed to create path for obj %s:%s.\n",
303 repo_id, obj_id);
304 return -1;
305 }
306
307 if (save_obj_contents (path, data, len, need_sync) < 0) {
308 seaf_warning ("[obj backend] Failed to write obj %s:%s.\n",
309 repo_id, obj_id);
310 return -1;
311 }
312
313 /* g_get_current_time (&e); */
314
315 /* seaf_message ("write obj time: %ldus.\n", */
316 /* ((e.tv_sec*1000000+e.tv_usec) - (s.tv_sec*1000000+s.tv_usec))); */
317
318 return 0;
319 }
320
321 static gboolean
obj_backend_fs_exists(ObjBackend * bend,const char * repo_id,int version,const char * obj_id)322 obj_backend_fs_exists (ObjBackend *bend,
323 const char *repo_id,
324 int version,
325 const char *obj_id)
326 {
327 char path[SEAF_PATH_MAX];
328 SeafStat st;
329
330 id_to_path (bend->priv, obj_id, path, repo_id, version);
331
332 if (seaf_stat (path, &st) == 0)
333 return TRUE;
334
335 return FALSE;
336 }
337
338 static void
obj_backend_fs_delete(ObjBackend * bend,const char * repo_id,int version,const char * obj_id)339 obj_backend_fs_delete (ObjBackend *bend,
340 const char *repo_id,
341 int version,
342 const char *obj_id)
343 {
344 char path[SEAF_PATH_MAX];
345
346 id_to_path (bend->priv, obj_id, path, repo_id, version);
347 g_unlink (path);
348 }
349
350 static int
obj_backend_fs_foreach_obj(ObjBackend * bend,const char * repo_id,int version,SeafObjFunc process,void * user_data)351 obj_backend_fs_foreach_obj (ObjBackend *bend,
352 const char *repo_id,
353 int version,
354 SeafObjFunc process,
355 void *user_data)
356 {
357 FsPriv *priv = bend->priv;
358 char *obj_dir = NULL;
359 int dir_len;
360 GDir *dir1 = NULL, *dir2;
361 const char *dname1, *dname2;
362 char obj_id[128];
363 char path[SEAF_PATH_MAX], *pos;
364 int ret = 0;
365
366 #if defined MIGRATION || defined SEAFILE_CLIENT
367 if (version > 0)
368 obj_dir = g_build_filename (priv->obj_dir, repo_id, NULL);
369 #else
370 obj_dir = g_build_filename (priv->obj_dir, repo_id, NULL);
371 #endif
372 dir_len = strlen (obj_dir);
373
374 dir1 = g_dir_open (obj_dir, 0, NULL);
375 if (!dir1) {
376 goto out;
377 }
378
379 memcpy (path, obj_dir, dir_len);
380 pos = path + dir_len;
381
382 while ((dname1 = g_dir_read_name(dir1)) != NULL) {
383 snprintf (pos, sizeof(path) - dir_len, "/%s", dname1);
384
385 dir2 = g_dir_open (path, 0, NULL);
386 if (!dir2) {
387 seaf_warning ("Failed to open object dir %s.\n", path);
388 continue;
389 }
390
391 while ((dname2 = g_dir_read_name(dir2)) != NULL) {
392 snprintf (obj_id, sizeof(obj_id), "%s%s", dname1, dname2);
393 if (!process (repo_id, version, obj_id, user_data)) {
394 g_dir_close (dir2);
395 goto out;
396 }
397 }
398 g_dir_close (dir2);
399 }
400
401 out:
402 if (dir1)
403 g_dir_close (dir1);
404 g_free (obj_dir);
405
406 return ret;
407 }
408
409 static int
obj_backend_fs_copy(ObjBackend * bend,const char * src_repo_id,int src_version,const char * dst_repo_id,int dst_version,const char * obj_id)410 obj_backend_fs_copy (ObjBackend *bend,
411 const char *src_repo_id,
412 int src_version,
413 const char *dst_repo_id,
414 int dst_version,
415 const char *obj_id)
416 {
417 char src_path[SEAF_PATH_MAX];
418 char dst_path[SEAF_PATH_MAX];
419
420 id_to_path (bend->priv, obj_id, src_path, src_repo_id, src_version);
421 id_to_path (bend->priv, obj_id, dst_path, dst_repo_id, dst_version);
422
423 if (g_file_test (dst_path, G_FILE_TEST_EXISTS))
424 return 0;
425
426 if (create_parent_path (dst_path) < 0) {
427 seaf_warning ("Failed to create dst path %s for obj %s.\n",
428 dst_path, obj_id);
429 return -1;
430 }
431
432 #ifdef WIN32
433 if (!CreateHardLink (dst_path, src_path, NULL)) {
434 seaf_warning ("Failed to link %s to %s: %lu.\n",
435 src_path, dst_path, GetLastError());
436 return -1;
437 }
438 return 0;
439 #else
440 int ret = link (src_path, dst_path);
441 if (ret < 0 && errno != EEXIST) {
442 seaf_warning ("Failed to link %s to %s: %s.\n",
443 src_path, dst_path, strerror(errno));
444 return -1;
445 }
446 return ret;
447 #endif
448 }
449
450 static int
obj_backend_fs_remove_store(ObjBackend * bend,const char * store_id)451 obj_backend_fs_remove_store (ObjBackend *bend, const char *store_id)
452 {
453 FsPriv *priv = bend->priv;
454 char *obj_dir = NULL;
455 GDir *dir1, *dir2;
456 const char *dname1, *dname2;
457 char *path1, *path2;
458
459 obj_dir = g_build_filename (priv->obj_dir, store_id, NULL);
460
461 dir1 = g_dir_open (obj_dir, 0, NULL);
462 if (!dir1) {
463 g_free (obj_dir);
464 return 0;
465 }
466
467 while ((dname1 = g_dir_read_name(dir1)) != NULL) {
468 path1 = g_build_filename (obj_dir, dname1, NULL);
469
470 dir2 = g_dir_open (path1, 0, NULL);
471 if (!dir2) {
472 seaf_warning ("Failed to open obj dir %s.\n", path1);
473 g_dir_close (dir1);
474 g_free (path1);
475 g_free (obj_dir);
476 return -1;
477 }
478
479 while ((dname2 = g_dir_read_name(dir2)) != NULL) {
480 path2 = g_build_filename (path1, dname2, NULL);
481 g_unlink (path2);
482 g_free (path2);
483 }
484 g_dir_close (dir2);
485
486 g_rmdir (path1);
487 g_free (path1);
488 }
489
490 g_dir_close (dir1);
491 g_rmdir (obj_dir);
492 g_free (obj_dir);
493
494 return 0;
495 }
496
497 ObjBackend *
obj_backend_fs_new(const char * seaf_dir,const char * obj_type)498 obj_backend_fs_new (const char *seaf_dir, const char *obj_type)
499 {
500 ObjBackend *bend;
501 FsPriv *priv;
502
503 bend = g_new0(ObjBackend, 1);
504 priv = g_new0(FsPriv, 1);
505 bend->priv = priv;
506
507 priv->obj_dir = g_build_filename (seaf_dir, "storage", obj_type, NULL);
508 priv->dir_len = strlen (priv->obj_dir);
509
510 if (g_mkdir_with_parents (priv->obj_dir, 0777) < 0) {
511 seaf_warning ("[Obj Backend] Objects dir %s does not exist and"
512 " is unable to create\n", priv->obj_dir);
513 goto onerror;
514 }
515
516 bend->read = obj_backend_fs_read;
517 bend->write = obj_backend_fs_write;
518 bend->exists = obj_backend_fs_exists;
519 bend->delete = obj_backend_fs_delete;
520 bend->foreach_obj = obj_backend_fs_foreach_obj;
521 bend->copy = obj_backend_fs_copy;
522 bend->remove_store = obj_backend_fs_remove_store;
523
524 return bend;
525
526 onerror:
527 g_free (priv->obj_dir);
528 g_free (priv);
529 g_free (bend);
530
531 return NULL;
532 }
533