1 #include <pthread.h>
2 #include <jansson.h>
3 
4 #include "common.h"
5 #include <timer.h>
6 #include "utils.h"
7 #include "log.h"
8 #include "seafile-error.h"
9 #include "seafile-session.h"
10 #include "pack-dir.h"
11 #include "web-accesstoken-mgr.h"
12 #include "zip-download-mgr.h"
13 
14 #define MAX_ZIP_THREAD_NUM 5
15 #define SCAN_PROGRESS_INTERVAL 24 * 3600 // 1 day
16 #define PROGRESS_TTL 5 * 3600 // 5 hours
17 #define DEFAULT_MAX_DOWNLOAD_DIR_SIZE 100 * ((gint64)1 << 20) /* 100MB */
18 
19 typedef struct ZipDownloadMgrPriv {
20     pthread_mutex_t progress_lock;
21     GHashTable *progress_store;
22     GThreadPool *zip_tpool;
23     // Abnormal behavior lead to no download request for the zip finished progress,
24     // so related progress will not be removed,
25     // this timer is used to scan progress and remove invalid progress.
26     CcnetTimer *scan_progress_timer;
27 } ZipDownloadMgrPriv;
28 
29 void
free_progress(Progress * progress)30 free_progress (Progress *progress)
31 {
32     if (!progress)
33         return;
34 
35     if (g_file_test (progress->zip_file_path, G_FILE_TEST_EXISTS)) {
36         g_unlink (progress->zip_file_path);
37     }
38     g_free (progress->zip_file_path);
39     g_free (progress);
40 }
41 
42 typedef enum DownloadType {
43     DOWNLOAD_DIR,
44     DOWNLOAD_MULTI
45 } DownloadType;
46 
47 typedef struct DownloadObj {
48     char *token;
49     DownloadType type;
50     SeafRepo *repo;
51     char *user;
52     gboolean is_windows;
53     // download-dir: top dir name; download-multi: ""
54     char *dir_name;
55     // download-dir: obj_id; download-multi: dirent list
56     void *internal;
57     Progress *progress;
58 } DownloadObj;
59 
60 static void
free_download_obj(DownloadObj * obj)61 free_download_obj (DownloadObj *obj)
62 {
63     if (!obj)
64         return;
65 
66     g_free (obj->token);
67     seaf_repo_unref (obj->repo);
68     g_free (obj->user);
69     g_free (obj->dir_name);
70     if (obj->type == DOWNLOAD_DIR) {
71         g_free ((char *)obj->internal);
72     } else {
73         g_list_free_full ((GList *)obj->internal, (GDestroyNotify)seaf_dirent_free);
74     }
75     g_free (obj);
76 }
77 
78 static void
79 start_zip_task (gpointer data, gpointer user_data);
80 
81 static int
82 scan_progress (void *data);
83 
84 static int
85 get_download_file_count (DownloadObj *obj, GError **error);
86 
87 static gboolean
88 validate_download_size (DownloadObj *obj, GError **error);
89 
90 ZipDownloadMgr *
zip_download_mgr_new()91 zip_download_mgr_new ()
92 {
93     GError *error = NULL;
94     ZipDownloadMgr *mgr = g_new0 (ZipDownloadMgr, 1);
95     ZipDownloadMgrPriv *priv = g_new0 (ZipDownloadMgrPriv, 1);
96 
97     priv->zip_tpool = g_thread_pool_new (start_zip_task, priv, MAX_ZIP_THREAD_NUM, FALSE, &error);
98     if (!priv->zip_tpool) {
99         if (error) {
100             seaf_warning ("Failed to create zip task thread pool: %s.\n", error->message);
101             g_clear_error (&error);
102         } else {
103             seaf_warning ("Failed to create zip task thread pool.\n");
104         }
105         g_free (priv);
106         g_free (mgr);
107         return NULL;
108     }
109 
110     pthread_mutex_init (&priv->progress_lock, NULL);
111     priv->progress_store = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
112                                                   (GDestroyNotify)free_progress);
113     priv->scan_progress_timer = ccnet_timer_new (scan_progress, priv,
114                                                  SCAN_PROGRESS_INTERVAL * 1000);
115     mgr->priv = priv;
116 
117     return mgr;
118 }
119 
120 static void
remove_progress_by_token(ZipDownloadMgrPriv * priv,const char * token)121 remove_progress_by_token (ZipDownloadMgrPriv *priv, const char *token)
122 {
123     pthread_mutex_lock (&priv->progress_lock);
124     g_hash_table_remove (priv->progress_store, token);
125     pthread_mutex_unlock (&priv->progress_lock);
126 }
127 
128 static int
scan_progress(void * data)129 scan_progress (void *data)
130 {
131     time_t now = time(NULL);
132     ZipDownloadMgrPriv *priv = data;
133     GHashTableIter iter;
134     gpointer key, value;
135     Progress *progress;
136 
137     pthread_mutex_lock (&priv->progress_lock);
138 
139     g_hash_table_iter_init (&iter, priv->progress_store);
140     while (g_hash_table_iter_next (&iter, &key, &value)) {
141         progress = value;
142         if (now >= progress->expire_ts) {
143             g_hash_table_iter_remove (&iter);
144         }
145     }
146 
147     pthread_mutex_unlock (&priv->progress_lock);
148 
149     return TRUE;
150 }
151 
152 static SeafileCrypt *
get_seafile_crypt(SeafRepo * repo,const char * user)153 get_seafile_crypt (SeafRepo *repo, const char *user)
154 {
155     SeafileCryptKey *key = NULL;
156     char *key_hex, *iv_hex;
157     unsigned char enc_key[32], enc_iv[16];
158     SeafileCrypt *crypt = NULL;
159 
160     key = seaf_passwd_manager_get_decrypt_key (seaf->passwd_mgr,
161                                                repo->id, user);
162     if (!key) {
163         seaf_warning ("Failed to get derypt key for repo %.8s.\n", repo->id);
164         return NULL;
165     }
166 
167     g_object_get (key, "key", &key_hex, "iv", &iv_hex, NULL);
168     if (repo->enc_version == 1)
169         hex_to_rawdata (key_hex, enc_key, 16);
170     else
171         hex_to_rawdata (key_hex, enc_key, 32);
172     hex_to_rawdata (iv_hex, enc_iv, 16);
173     crypt = seafile_crypt_new (repo->enc_version, enc_key, enc_iv);
174     g_free (key_hex);
175     g_free (iv_hex);
176     g_object_unref (key);
177 
178     return crypt;
179 }
180 
181 static void
start_zip_task(gpointer data,gpointer user_data)182 start_zip_task (gpointer data, gpointer user_data)
183 {
184     DownloadObj *obj = data;
185     ZipDownloadMgrPriv *priv = user_data;
186     SeafRepo *repo = obj->repo;
187     SeafileCrypt *crypt = NULL;
188     int ret = 0;
189 
190     if (repo->encrypted) {
191         crypt = get_seafile_crypt (repo, obj->user);
192         if (!crypt) {
193             ret = -1;
194             goto out;
195         }
196     }
197 
198     if (!validate_download_size (obj, NULL)) {
199         ret = -1;
200         obj->progress->size_too_large = TRUE;
201         goto out;
202     }
203 
204     int file_count = get_download_file_count (obj, NULL);
205     if (file_count < 0) {
206         ret = -1;
207         goto out;
208     }
209     obj->progress->total = file_count;
210 
211     ret = pack_files (repo->store_id, repo->version, obj->dir_name,
212                       obj->internal, crypt, obj->is_windows, obj->progress);
213 
214 out:
215     if (crypt) {
216         g_free (crypt);
217     }
218     if (ret == -1 && !obj->progress->canceled &&
219         !obj->progress->size_too_large) {
220         remove_progress_by_token (priv, obj->token);
221     }
222     free_download_obj (obj);
223 }
224 
225 static int
parse_download_dir_data(DownloadObj * obj,const char * data)226 parse_download_dir_data (DownloadObj *obj, const char *data)
227 {
228     json_t *jobj;
229     json_error_t jerror;
230     const char *dir_name;
231     const char *obj_id;
232 
233     jobj = json_loadb (data, strlen(data), 0, &jerror);
234     if (!jobj) {
235         seaf_warning ("Failed to parse download dir data: %s.\n", jerror.text);
236         return -1;
237     }
238 
239     obj->is_windows = json_object_get_int_member (jobj, "is_windows");
240 
241     dir_name = json_object_get_string_member (jobj, "dir_name");
242     if (!dir_name || strcmp (dir_name, "") == 0) {
243         seaf_warning ("Invalid download dir data: miss dir_name filed.\n");
244         json_decref (jobj);
245         return -1;
246     }
247 
248     obj_id = json_object_get_string_member (jobj, "obj_id");
249     if (!obj_id || strcmp (obj_id, "") == 0) {
250         seaf_warning ("Invalid download dir data: miss obj_id filed.\n");
251         json_decref (jobj);
252         return -1;
253     }
254 
255     obj->dir_name = g_strdup (dir_name);
256     obj->internal = g_strdup (obj_id);
257 
258     json_decref (jobj);
259 
260     return 0;
261 }
262 
263 static int
parse_download_multi_data(DownloadObj * obj,const char * data)264 parse_download_multi_data (DownloadObj *obj, const char *data)
265 {
266     json_t *jobj;
267     SeafRepo *repo = obj->repo;
268     const char *tmp_parent_dir;
269     char *parent_dir;
270     json_t *name_array;
271     json_error_t jerror;
272     int i;
273     int len;
274     const char *file_name;
275     SeafDirent *dirent;
276     SeafDir *dir;
277     GList *dirent_list = NULL, *p = NULL;
278     GError *error = NULL;
279 
280     jobj = json_loadb (data, strlen(data), 0, &jerror);
281     if (!jobj) {
282         seaf_warning ("Failed to parse download multi data: %s.\n", jerror.text);
283         return -1;
284     }
285 
286     obj->is_windows = json_object_get_int_member (jobj, "is_windows");
287 
288     tmp_parent_dir = json_object_get_string_member (jobj, "parent_dir");
289     if (!tmp_parent_dir || strcmp (tmp_parent_dir, "") == 0) {
290         seaf_warning ("Invalid download multi data, miss parent_dir field.\n");
291         json_decref (jobj);
292         return -1;
293     }
294     name_array = json_object_get (jobj, "file_list");
295     if (!name_array) {
296         seaf_warning ("Invalid download multi data, miss file_list field.\n");
297         json_decref (jobj);
298         return -1;
299     }
300     len = json_array_size (name_array);
301     if (len == 0) {
302         seaf_warning ("Invalid download multi data, miss download file name.\n");
303         json_decref (jobj);
304         return -1;
305     }
306     parent_dir = format_dir_path (tmp_parent_dir);
307 
308     dir = seaf_fs_manager_get_seafdir_by_path (seaf->fs_mgr, repo->store_id,
309                                                repo->version, repo->root_id, parent_dir, &error);
310     if (!dir) {
311         if (error) {
312             seaf_warning ("Failed to get dir %s repo %.8s: %s.\n",
313                           parent_dir, repo->store_id, error->message);
314             g_clear_error(&error);
315         } else {
316             seaf_warning ("dir %s doesn't exist in repo %.8s.\n",
317                           parent_dir, repo->store_id);
318         }
319         g_free (parent_dir);
320         json_decref (jobj);
321         return -1;
322     }
323     GHashTable *dirent_hash = g_hash_table_new(g_str_hash, g_str_equal);
324     for (p = dir->entries; p; p = p->next) {
325         SeafDirent *d = p->data;
326         g_hash_table_insert(dirent_hash, d->name, d);
327     }
328 
329     for (i = 0; i < len; i++) {
330         file_name = json_string_value (json_array_get (name_array, i));
331         if (strcmp (file_name, "") == 0 || strchr (file_name, '/') != NULL) {
332             seaf_warning ("Invalid download file name: %s.\n", file_name);
333             if (dirent_list) {
334                 g_list_free_full (dirent_list, (GDestroyNotify)seaf_dirent_free);
335                 dirent_list = NULL;
336             }
337             break;
338         }
339 
340         dirent = g_hash_table_lookup (dirent_hash, file_name);
341         if (!dirent) {
342             seaf_warning ("Failed to get dirent for %s in dir %s in repo %.8s.\n",
343                            file_name, parent_dir, repo->store_id);
344             if (dirent_list) {
345                 g_list_free_full (dirent_list, (GDestroyNotify)seaf_dirent_free);
346                 dirent_list = NULL;
347             }
348             break;
349         }
350 
351         dirent_list = g_list_prepend (dirent_list, seaf_dirent_dup(dirent));
352     }
353 
354     g_hash_table_unref(dirent_hash);
355     g_free (parent_dir);
356     json_decref (jobj);
357     seaf_dir_free (dir);
358 
359     if (!dirent_list) {
360         return -1;
361     }
362     obj->dir_name = g_strdup ("");
363     obj->internal = dirent_list;
364     return 0;
365 }
366 
367 static gint64
calcuate_download_multi_size(SeafRepo * repo,GList * dirent_list)368 calcuate_download_multi_size (SeafRepo *repo, GList *dirent_list)
369 {
370     GList *iter = dirent_list;
371     SeafDirent *dirent;
372     gint64 size;
373     gint64 total_size = 0;
374 
375     for (; iter; iter = iter->next) {
376         dirent = iter->data;
377         if (S_ISREG(dirent->mode)) {
378             if (repo->version > 0) {
379                 size = dirent->size;
380             } else {
381                 size = seaf_fs_manager_get_file_size (seaf->fs_mgr, repo->store_id,
382                                                       repo->version, dirent->id);
383             }
384             if (size < 0) {
385                 seaf_warning ("Failed to get file %s size.\n", dirent->name);
386                 return -1;
387             }
388             total_size += size;
389         } else if (S_ISDIR(dirent->mode)) {
390             size = seaf_fs_manager_get_fs_size (seaf->fs_mgr, repo->store_id,
391                                                 repo->version, dirent->id);
392             if (size < 0) {
393                 seaf_warning ("Failed to get dir %s size.\n", dirent->name);
394                 return -1;
395             }
396             total_size += size;
397         }
398     }
399 
400     return total_size;
401 }
402 
403 static int
calcuate_download_multi_file_count(SeafRepo * repo,GList * dirent_list)404 calcuate_download_multi_file_count (SeafRepo *repo, GList *dirent_list)
405 {
406     GList *iter = dirent_list;
407     SeafDirent *dirent;
408     int cur_count;
409     int count = 0;
410 
411     for (; iter; iter = iter->next) {
412         dirent = iter->data;
413         if (S_ISREG(dirent->mode)) {
414             count += 1;
415         } else if (S_ISDIR(dirent->mode)) {
416             cur_count = seaf_fs_manager_count_fs_files (seaf->fs_mgr, repo->store_id,
417                                                         repo->version, dirent->id);
418             if (cur_count < 0) {
419                 seaf_warning ("Failed to get dir %s file count.\n", dirent->name);
420                 return -1;
421             }
422             count += cur_count;
423         }
424     }
425 
426     return count;
427 }
428 
429 static gboolean
validate_download_size(DownloadObj * obj,GError ** error)430 validate_download_size (DownloadObj *obj, GError **error)
431 {
432     SeafRepo *repo = obj->repo;
433     gint64 download_size;
434     gint64 max_download_dir_size;
435 
436     if (obj->type == DOWNLOAD_DIR) {
437         download_size = seaf_fs_manager_get_fs_size (seaf->fs_mgr,
438                                                      repo->store_id, repo->version,
439                                                      (char *)obj->internal);
440     } else {
441         download_size = calcuate_download_multi_size (repo, (GList *)obj->internal);
442     }
443 
444     /* default is MB */
445     max_download_dir_size = seaf_cfg_manager_get_config_int64 (seaf->cfg_mgr, "fileserver",
446                                                                "max_download_dir_size");
447     if (max_download_dir_size > 0)
448         max_download_dir_size = max_download_dir_size * ((gint64)1 << 20);
449     else
450         max_download_dir_size = DEFAULT_MAX_DOWNLOAD_DIR_SIZE;
451 
452     if (download_size < 0) {
453         seaf_warning ("Failed to get download size.\n");
454         g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
455                      "Failed to get download size.");
456         return FALSE;
457     } else if (download_size > max_download_dir_size) {
458         seaf_warning ("Total download size %"G_GINT64_FORMAT
459                       ", exceed max download dir size %"G_GINT64_FORMAT".\n",
460                       download_size, max_download_dir_size);
461         g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
462                      "Download size exceed max download dir size.");
463         return FALSE;
464     }
465 
466     return TRUE;
467 }
468 
469 static int
get_download_file_count(DownloadObj * obj,GError ** error)470 get_download_file_count (DownloadObj *obj, GError **error)
471 {
472     int file_count;
473     SeafRepo *repo = obj->repo;
474 
475     if (obj->type == DOWNLOAD_DIR) {
476         file_count = seaf_fs_manager_count_fs_files (seaf->fs_mgr, repo->store_id,
477                                                      repo->version, (char *)obj->internal);
478     } else {
479         file_count = calcuate_download_multi_file_count (repo, (GList *)obj->internal);
480     }
481 
482     if (file_count < 0) {
483         seaf_warning ("Failed to get download file count.\n");
484         g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
485                      "Failed to get download file count.");
486         return -1;
487     }
488 
489     return file_count;
490 }
491 
492 int
zip_download_mgr_start_zip_task(ZipDownloadMgr * mgr,const char * token,SeafileWebAccess * info,GError ** error)493 zip_download_mgr_start_zip_task (ZipDownloadMgr *mgr,
494                                  const char *token,
495                                  SeafileWebAccess *info,
496                                  GError **error)
497 {
498     const char *repo_id;
499     const char *data;
500     const char *operation;
501     SeafRepo *repo;
502     DownloadObj *obj;
503     Progress *progress;
504     int ret = 0;
505     ZipDownloadMgrPriv *priv = mgr->priv;
506 
507     repo_id = seafile_web_access_get_repo_id (info);
508     repo = seaf_repo_manager_get_repo (seaf->repo_mgr, repo_id);
509     if (!repo) {
510         seaf_warning ("Failed to get repo %.8s.\n", repo_id);
511         g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
512                      "Failed to get repo.");
513         return -1;
514     }
515     data = seafile_web_access_get_obj_id (info);
516     operation = seafile_web_access_get_op (info);
517 
518     obj = g_new0 (DownloadObj, 1);
519     obj->token = g_strdup (token);
520     obj->repo = repo;
521     obj->user = g_strdup (seafile_web_access_get_username (info));
522 
523     if (strcmp (operation, "download-dir") == 0 ||
524         strcmp (operation, "download-dir-link") == 0) {
525         obj->type = DOWNLOAD_DIR;
526         ret = parse_download_dir_data (obj, data);
527         if (ret < 0) {
528             g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
529                          "Failed to parse download dir data.");
530             goto out;
531         }
532         if (!seaf_fs_manager_object_exists (seaf->fs_mgr,
533                                             repo->store_id, repo->version,
534                                             (char *)obj->internal)) {
535             seaf_warning ("Dir %s doesn't exist.\n", (char *)obj->internal);
536             g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
537                          "Dir doesn't exist.");
538             ret = -1;
539             goto out;
540         }
541     } else {
542         obj->type = DOWNLOAD_MULTI;
543         ret = parse_download_multi_data (obj, data);
544         if (ret < 0) {
545             g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
546                          "Failed to parse download multi data.");
547             goto out;
548         }
549     }
550 
551     progress = g_new0 (Progress, 1);
552     /* Set to real total in worker thread. Here to just prevent the client from thinking
553      * the zip has been finished too early.
554      */
555     progress->total = 1;
556     progress->expire_ts = time(NULL) + PROGRESS_TTL;
557     obj->progress = progress;
558 
559     pthread_mutex_lock (&priv->progress_lock);
560     g_hash_table_replace (priv->progress_store, g_strdup (token), progress);
561     pthread_mutex_unlock (&priv->progress_lock);
562 
563     g_thread_pool_push (priv->zip_tpool, obj, NULL);
564 
565 out:
566     if (ret < 0) {
567         free_download_obj (obj);
568     }
569 
570     return ret;
571 }
572 
573 static Progress *
get_progress_obj(ZipDownloadMgrPriv * priv,const char * token)574 get_progress_obj (ZipDownloadMgrPriv *priv, const char *token)
575 {
576     Progress *progress;
577 
578     pthread_mutex_lock (&priv->progress_lock);
579     progress = g_hash_table_lookup (priv->progress_store, token);
580     pthread_mutex_unlock (&priv->progress_lock);
581 
582     return progress;
583 }
584 
585 char *
zip_download_mgr_query_zip_progress(ZipDownloadMgr * mgr,const char * token,GError ** error)586 zip_download_mgr_query_zip_progress (ZipDownloadMgr *mgr,
587                                      const char *token, GError **error)
588 {
589     Progress *progress;
590     json_t *obj;
591     char *info;
592 
593     progress = get_progress_obj (mgr->priv, token);
594     if (!progress)
595         return NULL;
596 
597     obj = json_object ();
598     json_object_set_int_member (obj, "zipped", g_atomic_int_get (&progress->zipped));
599     json_object_set_int_member (obj, "total", progress->total);
600     if (progress->size_too_large) {
601         json_object_set_int_member (obj, "failed", 1);
602         json_object_set_string_member (obj, "failed_reason", "size too large");
603     } else {
604         json_object_set_int_member (obj, "failed", 0);
605         json_object_set_string_member (obj, "failed_reason", "");
606     }
607     if (progress->canceled)
608         json_object_set_int_member (obj, "canceled", 1);
609     else
610         json_object_set_int_member (obj, "canceled", 0);
611 
612     if (progress->size_too_large || progress->canceled)
613         remove_progress_by_token(mgr->priv, token);
614 
615     info = json_dumps (obj, JSON_COMPACT);
616     json_decref (obj);
617 
618     return info;
619 }
620 
621 char *
zip_download_mgr_get_zip_file_path(struct ZipDownloadMgr * mgr,const char * token)622 zip_download_mgr_get_zip_file_path (struct ZipDownloadMgr *mgr,
623                                     const char *token)
624 {
625     Progress *progress;
626 
627     progress = get_progress_obj (mgr->priv, token);
628     if (!progress) {
629         return NULL;
630     }
631     return progress->zip_file_path;
632 }
633 
634 void
zip_download_mgr_del_zip_progress(ZipDownloadMgr * mgr,const char * token)635 zip_download_mgr_del_zip_progress (ZipDownloadMgr *mgr,
636                                    const char *token)
637 {
638     remove_progress_by_token (mgr->priv, token);
639 }
640 
641 int
zip_download_mgr_cancel_zip_task(ZipDownloadMgr * mgr,const char * token)642 zip_download_mgr_cancel_zip_task (ZipDownloadMgr *mgr,
643                                   const char *token)
644 {
645     Progress *progress = get_progress_obj (mgr->priv, token);
646     if (progress)
647         progress->canceled = TRUE;
648 
649     return 0;
650 }
651