1 /******************************************************
2 hot backup tool for InnoDB
3 (c) 2009-2015 Percona LLC and/or its affiliates
4 (c) 2017 MariaDB
5 Originally Created 3/3/2009 Yasufumi Kinoshita
6 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
7 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; version 2 of the License.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
21
22 *******************************************************
23
24 This file incorporates work covered by the following copyright and
25 permission notice:
26
27 Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
28
29 This program is free software; you can redistribute it and/or modify it under
30 the terms of the GNU General Public License as published by the Free Software
31 Foundation; version 2 of the License.
32
33 This program is distributed in the hope that it will be useful, but WITHOUT
34 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
35 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
36
37 You should have received a copy of the GNU General Public License along with
38 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
39 Street, Fifth Floor, Boston, MA 02110-1335 USA
40
41 *******************************************************/
42
43 #include <my_global.h>
44 #include <os0file.h>
45 #include <my_dir.h>
46 #include <ut0mem.h>
47 #include <srv0start.h>
48 #include <fil0fil.h>
49 #include <trx0sys.h>
50 #include <set>
51 #include <string>
52 #include <mysqld.h>
53 #include <sstream>
54 #include "fil_cur.h"
55 #include "xtrabackup.h"
56 #include "common.h"
57 #include "backup_copy.h"
58 #include "backup_mysql.h"
59 #include <btr0btr.h>
60
61 #define ROCKSDB_BACKUP_DIR "#rocksdb"
62
63 /* list of files to sync for --rsync mode */
64 static std::set<std::string> rsync_list;
65 /* locations of tablespaces read from .isl files */
66 static std::map<std::string, std::string> tablespace_locations;
67
68 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
69 bool binlog_locked;
70
71 static void rocksdb_create_checkpoint();
72 static bool has_rocksdb_plugin();
73 static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
74 static void rocksdb_backup_checkpoint();
75 static void rocksdb_copy_back();
76
is_abs_path(const char * path)77 static bool is_abs_path(const char *path)
78 {
79 #ifdef _WIN32
80 return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
81 #else
82 return path[0] == '/';
83 #endif
84 }
85
86 /************************************************************************
87 Struct represents file or directory. */
88 struct datadir_node_t {
89 ulint dbpath_len;
90 char *filepath;
91 ulint filepath_len;
92 char *filepath_rel;
93 ulint filepath_rel_len;
94 bool is_empty_dir;
95 bool is_file;
96 };
97
98 /************************************************************************
99 Holds the state needed to enumerate files in MySQL data directory. */
100 struct datadir_iter_t {
101 char *datadir_path;
102 char *dbpath;
103 ulint dbpath_len;
104 char *filepath;
105 ulint filepath_len;
106 char *filepath_rel;
107 ulint filepath_rel_len;
108 pthread_mutex_t mutex;
109 os_file_dir_t dir;
110 os_file_dir_t dbdir;
111 os_file_stat_t dbinfo;
112 os_file_stat_t fileinfo;
113 dberr_t err;
114 bool is_empty_dir;
115 bool is_file;
116 bool skip_first_level;
117 };
118
119
120 /************************************************************************
121 Represents the context of the thread processing MySQL data directory. */
122 struct datadir_thread_ctxt_t {
123 datadir_iter_t *it;
124 uint n_thread;
125 uint *count;
126 pthread_mutex_t* count_mutex;
127 os_thread_id_t id;
128 bool ret;
129 };
130
131 static bool backup_files_from_datadir(const char *dir_path);
132
133 /************************************************************************
134 Retirn true if character if file separator */
135 bool
is_path_separator(char c)136 is_path_separator(char c)
137 {
138 return(c == FN_LIBCHAR || c == FN_LIBCHAR2);
139 }
140
141
142 /************************************************************************
143 Fill the node struct. Memory for node need to be allocated and freed by
144 the caller. It is caller responsibility to initialize node with
145 datadir_node_init and cleanup the memory with datadir_node_free.
146 Node can not be shared between threads. */
147 static
148 void
datadir_node_fill(datadir_node_t * node,datadir_iter_t * it)149 datadir_node_fill(datadir_node_t *node, datadir_iter_t *it)
150 {
151 if (node->filepath_len < it->filepath_len) {
152 free(node->filepath);
153 node->filepath = (char*)(malloc(it->filepath_len));
154 node->filepath_len = it->filepath_len;
155 }
156 if (node->filepath_rel_len < it->filepath_rel_len) {
157 free(node->filepath_rel);
158 node->filepath_rel = (char*)(malloc(it->filepath_rel_len));
159 node->filepath_rel_len = it->filepath_rel_len;
160 }
161
162 strcpy(node->filepath, it->filepath);
163 strcpy(node->filepath_rel, it->filepath_rel);
164 node->is_empty_dir = it->is_empty_dir;
165 node->is_file = it->is_file;
166 }
167
168 static
169 void
datadir_node_free(datadir_node_t * node)170 datadir_node_free(datadir_node_t *node)
171 {
172 free(node->filepath);
173 free(node->filepath_rel);
174 memset(node, 0, sizeof(datadir_node_t));
175 }
176
177 static
178 void
datadir_node_init(datadir_node_t * node)179 datadir_node_init(datadir_node_t *node)
180 {
181 memset(node, 0, sizeof(datadir_node_t));
182 }
183
184
185 /************************************************************************
186 Create the MySQL data directory iterator. Memory needs to be released
187 with datadir_iter_free. Position should be advanced with
188 datadir_iter_next_file. Iterator can be shared between multiple
189 threads. It is guaranteed that each thread receives unique file from
190 data directory into its local node struct. */
191 static
192 datadir_iter_t *
datadir_iter_new(const char * path,bool skip_first_level=true)193 datadir_iter_new(const char *path, bool skip_first_level = true)
194 {
195 datadir_iter_t *it;
196
197 it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t)));
198 if (!it)
199 goto error;
200 memset(it, 0, sizeof(datadir_iter_t));
201
202 pthread_mutex_init(&it->mutex, NULL);
203 it->datadir_path = strdup(path);
204
205 it->dir = os_file_opendir(it->datadir_path);
206
207 if (it->dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
208
209 goto error;
210 }
211
212 it->err = DB_SUCCESS;
213
214 it->dbpath_len = FN_REFLEN;
215 it->dbpath = static_cast<char*>(malloc(it->dbpath_len));
216
217 it->filepath_len = FN_REFLEN;
218 it->filepath = static_cast<char*>(malloc(it->filepath_len));
219
220 it->filepath_rel_len = FN_REFLEN;
221 it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len));
222
223 it->skip_first_level = skip_first_level;
224
225 return(it);
226
227 error:
228 free(it);
229
230 return(NULL);
231 }
232
233 static
234 bool
datadir_iter_next_database(datadir_iter_t * it)235 datadir_iter_next_database(datadir_iter_t *it)
236 {
237 if (it->dbdir != NULL) {
238 if (os_file_closedir_failed(it->dbdir)) {
239 msg("Warning: could not"
240 " close database directory %s", it->dbpath);
241 it->err = DB_ERROR;
242
243 }
244 it->dbdir = NULL;
245 }
246
247 while (os_file_readdir_next_file(it->datadir_path,
248 it->dir, &it->dbinfo) == 0) {
249 ulint len;
250
251 if ((it->dbinfo.type == OS_FILE_TYPE_FILE
252 && it->skip_first_level)
253 || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
254
255 continue;
256 }
257
258 /* We found a symlink or a directory; try opening it to see
259 if a symlink is a directory */
260
261 len = strlen(it->datadir_path)
262 + strlen (it->dbinfo.name) + 2;
263 if (len > it->dbpath_len) {
264 it->dbpath_len = len;
265 free(it->dbpath);
266
267 it->dbpath = static_cast<char*>(
268 malloc(it->dbpath_len));
269 }
270 snprintf(it->dbpath, it->dbpath_len, "%s/%s",
271 it->datadir_path, it->dbinfo.name);
272 os_normalize_path(it->dbpath);
273
274 if (it->dbinfo.type == OS_FILE_TYPE_FILE) {
275 it->is_file = true;
276 return(true);
277 }
278
279 if (check_if_skip_database_by_path(it->dbpath)) {
280 msg("Skipping db: %s", it->dbpath);
281 continue;
282 }
283
284 /* We want wrong directory permissions to be a fatal error for
285 XtraBackup. */
286 it->dbdir = os_file_opendir(it->dbpath);
287
288 if (it->dir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
289 it->is_file = false;
290 return(true);
291 }
292
293 }
294
295 return(false);
296 }
297
298 /************************************************************************
299 Concatenate n parts into single path */
300 static
301 void
make_path_n(int n,char ** path,ulint * path_len,...)302 make_path_n(int n, char **path, ulint *path_len, ...)
303 {
304 ulint len_needed = n + 1;
305 char *p;
306 int i;
307 va_list vl;
308
309 ut_ad(n > 0);
310
311 va_start(vl, path_len);
312 for (i = 0; i < n; i++) {
313 p = va_arg(vl, char*);
314 len_needed += strlen(p);
315 }
316 va_end(vl);
317
318 if (len_needed < *path_len) {
319 free(*path);
320 *path = static_cast<char*>(malloc(len_needed));
321 }
322
323 va_start(vl, path_len);
324 p = va_arg(vl, char*);
325 strcpy(*path, p);
326 for (i = 1; i < n; i++) {
327 size_t plen;
328 p = va_arg(vl, char*);
329 plen = strlen(*path);
330 if (!is_path_separator((*path)[plen - 1])) {
331 (*path)[plen] = FN_LIBCHAR;
332 (*path)[plen + 1] = 0;
333 }
334 strcat(*path + plen, p);
335 }
336 va_end(vl);
337 }
338
339 static
340 bool
datadir_iter_next_file(datadir_iter_t * it)341 datadir_iter_next_file(datadir_iter_t *it)
342 {
343 if (it->is_file && it->dbpath) {
344 make_path_n(2, &it->filepath, &it->filepath_len,
345 it->datadir_path, it->dbinfo.name);
346
347 make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
348 it->dbinfo.name);
349
350 it->is_empty_dir = false;
351 it->is_file = false;
352
353 return(true);
354 }
355
356 if (!it->dbpath || !it->dbdir) {
357
358 return(false);
359 }
360
361 while (os_file_readdir_next_file(it->dbpath, it->dbdir,
362 &it->fileinfo) == 0) {
363
364 if (it->fileinfo.type == OS_FILE_TYPE_DIR) {
365
366 continue;
367 }
368
369 /* We found a symlink or a file */
370 make_path_n(3, &it->filepath, &it->filepath_len,
371 it->datadir_path, it->dbinfo.name,
372 it->fileinfo.name);
373
374 make_path_n(2, &it->filepath_rel, &it->filepath_rel_len,
375 it->dbinfo.name, it->fileinfo.name);
376
377 it->is_empty_dir = false;
378
379 return(true);
380 }
381
382 return(false);
383 }
384
385 static
386 bool
datadir_iter_next(datadir_iter_t * it,datadir_node_t * node)387 datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
388 {
389 bool ret = true;
390
391 pthread_mutex_lock(&it->mutex);
392
393 if (datadir_iter_next_file(it)) {
394
395 datadir_node_fill(node, it);
396
397 goto done;
398 }
399
400 while (datadir_iter_next_database(it)) {
401
402 if (datadir_iter_next_file(it)) {
403
404 datadir_node_fill(node, it);
405
406 goto done;
407 }
408
409 make_path_n(2, &it->filepath, &it->filepath_len,
410 it->datadir_path, it->dbinfo.name);
411
412 make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
413 it->dbinfo.name);
414
415 it->is_empty_dir = true;
416
417 datadir_node_fill(node, it);
418
419 goto done;
420 }
421
422 /* nothing found */
423 ret = false;
424
425 done:
426 pthread_mutex_unlock(&it->mutex);
427
428 return(ret);
429 }
430
431 /************************************************************************
432 Interface to read MySQL data file sequentially. One should open file
433 with datafile_open to get cursor and close the cursor with
434 datafile_close. Cursor can not be shared between multiple
435 threads. */
436 static
437 void
datadir_iter_free(datadir_iter_t * it)438 datadir_iter_free(datadir_iter_t *it)
439 {
440 pthread_mutex_destroy(&it->mutex);
441
442 if (it->dbdir) {
443
444 os_file_closedir(it->dbdir);
445 }
446
447 if (it->dir) {
448
449 os_file_closedir(it->dir);
450 }
451
452 free(it->dbpath);
453 free(it->filepath);
454 free(it->filepath_rel);
455 free(it->datadir_path);
456 free(it);
457 }
458
459
460 /************************************************************************
461 Holds the state needed to copy single data file. */
462 struct datafile_cur_t {
463 pfs_os_file_t file;
464 char rel_path[FN_REFLEN];
465 char abs_path[FN_REFLEN];
466 MY_STAT statinfo;
467 uint thread_n;
468 byte* orig_buf;
469 byte* buf;
470 size_t buf_size;
471 size_t buf_read;
472 size_t buf_offset;
473
datafile_cur_tdatafile_cur_t474 explicit datafile_cur_t(const char* filename = NULL) :
475 file(), thread_n(0), orig_buf(NULL), buf(NULL), buf_size(0),
476 buf_read(0), buf_offset(0)
477 {
478 memset(rel_path, 0, sizeof rel_path);
479 if (filename) {
480 strncpy(abs_path, filename, sizeof abs_path - 1);
481 abs_path[(sizeof abs_path) - 1] = 0;
482 } else {
483 abs_path[0] = '\0';
484 }
485 rel_path[0] = '\0';
486 memset(&statinfo, 0, sizeof statinfo);
487 }
488 };
489
490 static
491 void
datafile_close(datafile_cur_t * cursor)492 datafile_close(datafile_cur_t *cursor)
493 {
494 if (cursor->file != OS_FILE_CLOSED) {
495 os_file_close(cursor->file);
496 }
497 free(cursor->buf);
498 }
499
500 static
501 bool
datafile_open(const char * file,datafile_cur_t * cursor,uint thread_n)502 datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
503 {
504 bool success;
505
506 new (cursor) datafile_cur_t(file);
507
508 /* Get the relative path for the destination tablespace name, i.e. the
509 one that can be appended to the backup root directory. Non-system
510 tablespaces may have absolute paths for remote tablespaces in MySQL
511 5.6+. We want to make "local" copies for the backup. */
512 strncpy(cursor->rel_path,
513 xb_get_relative_path(cursor->abs_path, FALSE),
514 (sizeof cursor->rel_path) - 1);
515 cursor->rel_path[(sizeof cursor->rel_path) - 1] = '\0';
516
517 cursor->file = os_file_create_simple_no_error_handling(
518 0, cursor->abs_path,
519 OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success);
520 if (!success) {
521 /* The following call prints an error message */
522 os_file_get_last_error(TRUE);
523
524 msg(thread_n,"error: cannot open "
525 "file %s", cursor->abs_path);
526
527 return(false);
528 }
529
530 if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) {
531 msg(thread_n, "error: cannot stat %s", cursor->abs_path);
532 datafile_close(cursor);
533 return(false);
534 }
535
536 posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
537
538 cursor->buf_size = 10 * 1024 * 1024;
539 cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size));
540
541 return(true);
542 }
543
544
545 static
546 xb_fil_cur_result_t
datafile_read(datafile_cur_t * cursor)547 datafile_read(datafile_cur_t *cursor)
548 {
549 ulint to_read;
550
551 xtrabackup_io_throttling();
552
553 to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset,
554 cursor->buf_size);
555
556 if (to_read == 0) {
557 return(XB_FIL_CUR_EOF);
558 }
559
560 if (os_file_read(IORequestRead,
561 cursor->file, cursor->buf, cursor->buf_offset,
562 to_read) != DB_SUCCESS) {
563 return(XB_FIL_CUR_ERROR);
564 }
565
566 posix_fadvise(cursor->file, cursor->buf_offset, to_read,
567 POSIX_FADV_DONTNEED);
568
569 cursor->buf_read = to_read;
570 cursor->buf_offset += to_read;
571
572 return(XB_FIL_CUR_SUCCESS);
573 }
574
575
576
577 /************************************************************************
578 Check to see if a file exists.
579 Takes name of the file to check.
580 @return true if file exists. */
581 static
582 bool
file_exists(const char * filename)583 file_exists(const char *filename)
584 {
585 MY_STAT stat_arg;
586
587 if (!my_stat(filename, &stat_arg, MYF(0))) {
588
589 return(false);
590 }
591
592 return(true);
593 }
594
595 /************************************************************************
596 Trim leading slashes from absolute path so it becomes relative */
597 static
598 const char *
trim_dotslash(const char * path)599 trim_dotslash(const char *path)
600 {
601 while (*path) {
602 if (is_path_separator(*path)) {
603 ++path;
604 continue;
605 }
606 if (*path == '.' && is_path_separator(path[1])) {
607 path += 2;
608 continue;
609 }
610 break;
611 }
612
613 return(path);
614 }
615
616
617
618 /************************************************************************
619 Check if string ends with given suffix.
620 @return true if string ends with given suffix. */
621 bool
ends_with(const char * str,const char * suffix)622 ends_with(const char *str, const char *suffix)
623 {
624 size_t suffix_len = strlen(suffix);
625 size_t str_len = strlen(str);
626 return(str_len >= suffix_len
627 && strcmp(str + str_len - suffix_len, suffix) == 0);
628 }
629
starts_with(const char * str,const char * prefix)630 static bool starts_with(const char *str, const char *prefix)
631 {
632 return strncmp(str, prefix, strlen(prefix)) == 0;
633 }
634
635 /************************************************************************
636 Create directories recursively.
637 @return 0 if directories created successfully. */
638 static
639 int
mkdirp(const char * pathname,int Flags,myf MyFlags)640 mkdirp(const char *pathname, int Flags, myf MyFlags)
641 {
642 char *parent, *p;
643
644 /* make a parent directory path */
645 if (!(parent= strdup(pathname)))
646 return(-1);
647
648 for (p = parent + strlen(parent);
649 !is_path_separator(*p) && p != parent; p--) ;
650
651 *p = 0;
652
653 /* try to make parent directory */
654 if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) {
655 free(parent);
656 return(-1);
657 }
658
659 /* make this one if parent has been made */
660 if (my_mkdir(pathname, Flags, MyFlags) == 0) {
661 free(parent);
662 return(0);
663 }
664
665 /* if it already exists that is fine */
666 if (errno == EEXIST) {
667 free(parent);
668 return(0);
669 }
670
671 free(parent);
672 return(-1);
673 }
674
675 /************************************************************************
676 Return true if first and second arguments are the same path. */
677 bool
equal_paths(const char * first,const char * second)678 equal_paths(const char *first, const char *second)
679 {
680 #ifdef HAVE_REALPATH
681 char *real_first, *real_second;
682 int result;
683
684 real_first = realpath(first, 0);
685 if (real_first == NULL) {
686 return false;
687 }
688
689 real_second = realpath(second, 0);
690 if (real_second == NULL) {
691 free(real_first);
692 return false;
693 }
694
695 result = strcmp(real_first, real_second);
696 free(real_first);
697 free(real_second);
698 return result == 0;
699 #else
700 return strcmp(first, second) == 0;
701 #endif
702 }
703
704 /************************************************************************
705 Check if directory exists. Optionally create directory if doesn't
706 exist.
707 @return true if directory exists and if it was created successfully. */
708 bool
directory_exists(const char * dir,bool create)709 directory_exists(const char *dir, bool create)
710 {
711 os_file_dir_t os_dir;
712 MY_STAT stat_arg;
713 char errbuf[MYSYS_STRERROR_SIZE];
714
715 if (my_stat(dir, &stat_arg, MYF(0)) == NULL) {
716
717 if (!create) {
718 return(false);
719 }
720
721 if (mkdirp(dir, 0777, MYF(0)) < 0) {
722 my_strerror(errbuf, sizeof(errbuf), my_errno);
723 msg("Can not create directory %s: %s", dir, errbuf);
724 return(false);
725 }
726 }
727
728 /* could be symlink */
729 os_dir = os_file_opendir(dir);
730
731 if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
732 my_strerror(errbuf, sizeof(errbuf), my_errno);
733 msg("Can not open directory %s: %s", dir,
734 errbuf);
735
736 return(false);
737 }
738
739 os_file_closedir(os_dir);
740
741 return(true);
742 }
743
744 /************************************************************************
745 Check that directory exists and it is empty. */
746 static
747 bool
directory_exists_and_empty(const char * dir,const char * comment)748 directory_exists_and_empty(const char *dir, const char *comment)
749 {
750 os_file_dir_t os_dir;
751 dberr_t err;
752 os_file_stat_t info;
753 bool empty;
754
755 if (!directory_exists(dir, true)) {
756 return(false);
757 }
758
759 os_dir = os_file_opendir(dir);
760
761 if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
762 msg("%s can not open directory %s", comment, dir);
763 return(false);
764 }
765
766 empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0);
767
768 os_file_closedir(os_dir);
769
770 if (!empty) {
771 msg("%s directory %s is not empty!", comment, dir);
772 }
773
774 return(empty);
775 }
776
777
778 /************************************************************************
779 Check if file name ends with given set of suffixes.
780 @return true if it does. */
781 static
782 bool
filename_matches(const char * filename,const char ** ext_list)783 filename_matches(const char *filename, const char **ext_list)
784 {
785 const char **ext;
786
787 for (ext = ext_list; *ext; ext++) {
788 if (ends_with(filename, *ext)) {
789 return(true);
790 }
791 }
792
793 return(false);
794 }
795
796
797 /************************************************************************
798 Copy data file for backup. Also check if it is allowed to copy by
799 comparing its name to the list of known data file types and checking
800 if passes the rules for partial backup.
801 @return true if file backed up or skipped successfully. */
802 static
803 bool
datafile_copy_backup(const char * filepath,uint thread_n)804 datafile_copy_backup(const char *filepath, uint thread_n)
805 {
806 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
807 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
808 NULL};
809
810 /* Get the name and the path for the tablespace. node->name always
811 contains the path (which may be absolute for remote tablespaces in
812 5.6+). space->name contains the tablespace name in the form
813 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
814 multi-node shared tablespace, space->name contains the name of the first
815 node, but that's irrelevant, since we only need node_name to match them
816 against filters, and the shared tablespace is always copied regardless
817 of the filters value. */
818
819 if (check_if_skip_table(filepath)) {
820 msg(thread_n,"Skipping %s.", filepath);
821 return(true);
822 }
823
824 if (filename_matches(filepath, ext_list)) {
825 return copy_file(ds_data, filepath, filepath, thread_n);
826 }
827
828 return(true);
829 }
830
831
832 /************************************************************************
833 Same as datafile_copy_backup, but put file name into the list for
834 rsync command. */
835 static
836 bool
datafile_rsync_backup(const char * filepath,bool save_to_list,FILE * f)837 datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
838 {
839 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
840 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
841 NULL};
842
843 /* Get the name and the path for the tablespace. node->name always
844 contains the path (which may be absolute for remote tablespaces in
845 5.6+). space->name contains the tablespace name in the form
846 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
847 multi-node shared tablespace, space->name contains the name of the first
848 node, but that's irrelevant, since we only need node_name to match them
849 against filters, and the shared tablespace is always copied regardless
850 of the filters value. */
851
852 if (check_if_skip_table(filepath)) {
853 return(true);
854 }
855
856 if (filename_matches(filepath, ext_list)) {
857 fprintf(f, "%s\n", filepath);
858 if (save_to_list) {
859 rsync_list.insert(filepath);
860 }
861 }
862
863 return(true);
864 }
865
backup_file_print_buf(const char * filename,const char * buf,int buf_len)866 bool backup_file_print_buf(const char *filename, const char *buf, int buf_len)
867 {
868 ds_file_t *dstfile = NULL;
869 MY_STAT stat; /* unused for now */
870 const char *action;
871
872 memset(&stat, 0, sizeof(stat));
873
874 stat.st_size = buf_len;
875 stat.st_mtime = my_time(0);
876
877 dstfile = ds_open(ds_data, filename, &stat);
878 if (dstfile == NULL) {
879 msg("error: Can't open the destination stream for %s",
880 filename);
881 goto error;
882 }
883
884 action = xb_get_copy_action("Writing");
885 msg("%s %s", action, filename);
886
887 if (buf_len == -1) {
888 goto error;
889 }
890
891 if (ds_write(dstfile, buf, buf_len)) {
892 goto error;
893 }
894
895 /* close */
896 msg(" ...done");
897
898 if (ds_close(dstfile)) {
899 goto error_close;
900 }
901
902 return(true);
903
904 error:
905 if (dstfile != NULL) {
906 ds_close(dstfile);
907 }
908
909 error_close:
910 msg("Error: backup file failed.");
911 return(false); /*ERROR*/
912
913 return true;
914 };
915
916 static
917 bool
backup_file_vprintf(const char * filename,const char * fmt,va_list ap)918 backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
919 {
920 char *buf = 0;
921 int buf_len;
922 buf_len = vasprintf(&buf, fmt, ap);
923 bool result = backup_file_print_buf(filename, buf, buf_len);
924 free(buf);
925 return result;
926 }
927
928 bool
backup_file_printf(const char * filename,const char * fmt,...)929 backup_file_printf(const char *filename, const char *fmt, ...)
930 {
931 bool result;
932 va_list ap;
933
934 va_start(ap, fmt);
935
936 result = backup_file_vprintf(filename, fmt, ap);
937
938 va_end(ap);
939
940 return(result);
941 }
942
943 static
944 bool
run_data_threads(datadir_iter_t * it,os_thread_func_t func,uint n)945 run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
946 {
947 datadir_thread_ctxt_t *data_threads;
948 uint i, count;
949 pthread_mutex_t count_mutex;
950 bool ret;
951
952 data_threads = (datadir_thread_ctxt_t*)
953 malloc(sizeof(datadir_thread_ctxt_t) * n);
954
955 pthread_mutex_init(&count_mutex, NULL);
956 count = n;
957
958 for (i = 0; i < n; i++) {
959 data_threads[i].it = it;
960 data_threads[i].n_thread = i + 1;
961 data_threads[i].count = &count;
962 data_threads[i].count_mutex = &count_mutex;
963 os_thread_create(func, data_threads + i, &data_threads[i].id);
964 }
965
966 /* Wait for threads to exit */
967 while (1) {
968 os_thread_sleep(100000);
969 pthread_mutex_lock(&count_mutex);
970 if (count == 0) {
971 pthread_mutex_unlock(&count_mutex);
972 break;
973 }
974 pthread_mutex_unlock(&count_mutex);
975 }
976
977 pthread_mutex_destroy(&count_mutex);
978
979 ret = true;
980 for (i = 0; i < n; i++) {
981 ret = data_threads[i].ret && ret;
982 if (!data_threads[i].ret) {
983 msg("Error: thread %u failed.", i);
984 }
985 }
986
987 free(data_threads);
988
989 return(ret);
990 }
991
992 #ifdef _WIN32
993 #include <windows.h>
994 #include <accctrl.h>
995 #include <aclapi.h>
996 /*
997 On Windows, fix permission of the file after "copyback"
998 We assume that after copyback, mysqld will run as service as NetworkService
999 user, thus well give full permission on given file to that user.
1000 */
1001
fix_win_file_permissions(const char * file)1002 static int fix_win_file_permissions(const char *file)
1003 {
1004 struct {
1005 TOKEN_USER tokenUser;
1006 BYTE buffer[SECURITY_MAX_SID_SIZE];
1007 } tokenInfoBuffer;
1008 HANDLE hFile = CreateFile(file, READ_CONTROL | WRITE_DAC, 0, NULL, OPEN_EXISTING,
1009 FILE_FLAG_BACKUP_SEMANTICS, NULL);
1010 if (hFile == INVALID_HANDLE_VALUE)
1011 return -1;
1012 ACL* pOldDACL;
1013 SECURITY_DESCRIPTOR* pSD = NULL;
1014 EXPLICIT_ACCESS ea = { 0 };
1015 PSID pSid = NULL;
1016
1017 GetSecurityInfo(hFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL,
1018 &pOldDACL, NULL, (void**)&pSD);
1019 DWORD size = SECURITY_MAX_SID_SIZE;
1020 pSid = (PSID)tokenInfoBuffer.buffer;
1021 if (!CreateWellKnownSid(WinNetworkServiceSid, NULL, pSid,
1022 &size))
1023 {
1024 return 1;
1025 }
1026 ea.Trustee.TrusteeForm = TRUSTEE_IS_SID;
1027 ea.Trustee.ptstrName = (LPTSTR)pSid;
1028
1029 ea.grfAccessMode = GRANT_ACCESS;
1030 ea.grfAccessPermissions = GENERIC_ALL;
1031 ea.grfInheritance = CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE;
1032 ea.Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
1033 ACL* pNewDACL = 0;
1034 SetEntriesInAcl(1, &ea, pOldDACL, &pNewDACL);
1035 if (pNewDACL)
1036 {
1037 SetSecurityInfo(hFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL,
1038 pNewDACL, NULL);
1039 }
1040 if (pSD != NULL)
1041 LocalFree((HLOCAL)pSD);
1042 if (pNewDACL != NULL)
1043 LocalFree((HLOCAL)pNewDACL);
1044 CloseHandle(hFile);
1045 return 0;
1046 }
1047
1048 #endif
1049
1050
1051 /************************************************************************
1052 Copy file for backup/restore.
1053 @return true in case of success. */
1054 bool
copy_file(ds_ctxt_t * datasink,const char * src_file_path,const char * dst_file_path,uint thread_n)1055 copy_file(ds_ctxt_t *datasink,
1056 const char *src_file_path,
1057 const char *dst_file_path,
1058 uint thread_n)
1059 {
1060 char dst_name[FN_REFLEN];
1061 ds_file_t *dstfile = NULL;
1062 datafile_cur_t cursor;
1063 xb_fil_cur_result_t res;
1064 DBUG_ASSERT(datasink->datasink->remove);
1065 const char *dst_path =
1066 (xtrabackup_copy_back || xtrabackup_move_back)?
1067 dst_file_path : trim_dotslash(dst_file_path);
1068
1069 if (!datafile_open(src_file_path, &cursor, thread_n)) {
1070 goto error_close;
1071 }
1072
1073 strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
1074
1075 dstfile = ds_open(datasink, dst_path, &cursor.statinfo);
1076 if (dstfile == NULL) {
1077 msg(thread_n,"error: "
1078 "cannot open the destination stream for %s", dst_name);
1079 goto error;
1080 }
1081
1082 msg(thread_n, "%s %s to %s", xb_get_copy_action(), src_file_path, dstfile->path);
1083
1084 /* The main copy loop */
1085 while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
1086
1087 if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
1088 goto error;
1089 }
1090 DBUG_EXECUTE_IF("copy_file_error", errno=ENOSPC;goto error;);
1091 }
1092
1093 if (res == XB_FIL_CUR_ERROR) {
1094 goto error;
1095 }
1096
1097 /* close */
1098 msg(thread_n," ...done");
1099 datafile_close(&cursor);
1100 #ifdef _WIN32
1101 if (xtrabackup_copy_back || xtrabackup_move_back)
1102 ut_a(!fix_win_file_permissions(dstfile->path));
1103 #endif
1104 if (ds_close(dstfile)) {
1105 goto error_close;
1106 }
1107 return(true);
1108
1109 error:
1110 datafile_close(&cursor);
1111 if (dstfile != NULL) {
1112 datasink->datasink->remove(dstfile->path);
1113 ds_close(dstfile);
1114 }
1115
1116 error_close:
1117 msg(thread_n,"Error: copy_file() failed.");
1118 return(false); /*ERROR*/
1119 }
1120
1121
1122 /************************************************************************
1123 Try to move file by renaming it. If source and destination are on
1124 different devices fall back to copy and unlink.
1125 @return true in case of success. */
1126 static
1127 bool
move_file(ds_ctxt_t * datasink,const char * src_file_path,const char * dst_file_path,const char * dst_dir,uint thread_n)1128 move_file(ds_ctxt_t *datasink,
1129 const char *src_file_path,
1130 const char *dst_file_path,
1131 const char *dst_dir, uint thread_n)
1132 {
1133 char errbuf[MYSYS_STRERROR_SIZE];
1134 char dst_file_path_abs[FN_REFLEN];
1135 char dst_dir_abs[FN_REFLEN];
1136 size_t dirname_length;
1137
1138 snprintf(dst_file_path_abs, sizeof(dst_file_path_abs),
1139 "%s/%s", dst_dir, dst_file_path);
1140
1141 dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length);
1142
1143 if (!directory_exists(dst_dir_abs, true)) {
1144 return(false);
1145 }
1146
1147 if (file_exists(dst_file_path_abs)) {
1148 msg("Error: Move file %s to %s failed: Destination "
1149 "file exists", src_file_path, dst_file_path_abs);
1150 return(false);
1151 }
1152
1153 msg(thread_n,"Moving %s to %s", src_file_path, dst_file_path_abs);
1154
1155 if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
1156 if (my_errno == EXDEV) {
1157 /* Fallback to copy/unlink */
1158 if(!copy_file(datasink, src_file_path,
1159 dst_file_path, thread_n))
1160 return false;
1161 msg(thread_n,"Removing %s", src_file_path);
1162 if (unlink(src_file_path) != 0) {
1163 my_strerror(errbuf, sizeof(errbuf), errno);
1164 msg("Warning: unlink %s failed: %s",
1165 src_file_path,
1166 errbuf);
1167 }
1168 return true;
1169 }
1170 my_strerror(errbuf, sizeof(errbuf), my_errno);
1171 msg("Can not move file %s to %s: %s",
1172 src_file_path, dst_file_path_abs,
1173 errbuf);
1174 return(false);
1175 }
1176 #ifdef _WIN32
1177 if (xtrabackup_copy_back || xtrabackup_move_back)
1178 ut_a(!fix_win_file_permissions(dst_file_path_abs));
1179 #endif
1180 msg(thread_n," ...done");
1181
1182 return(true);
1183 }
1184
1185
1186 /************************************************************************
1187 Read link from .isl file if any and store it in the global map associated
1188 with given tablespace. */
1189 static
1190 void
read_link_file(const char * ibd_filepath,const char * link_filepath)1191 read_link_file(const char *ibd_filepath, const char *link_filepath)
1192 {
1193 char *filepath= NULL;
1194
1195 FILE *file = fopen(link_filepath, "r+b");
1196 if (file) {
1197 filepath = static_cast<char*>(malloc(OS_FILE_MAX_PATH));
1198
1199 os_file_read_string(file, filepath, OS_FILE_MAX_PATH);
1200 fclose(file);
1201
1202 if (strlen(filepath)) {
1203 /* Trim whitespace from end of filepath */
1204 ulint lastch = strlen(filepath) - 1;
1205 while (lastch > 4 && filepath[lastch] <= 0x20) {
1206 filepath[lastch--] = 0x00;
1207 }
1208 os_normalize_path(filepath);
1209 }
1210
1211 tablespace_locations[ibd_filepath] = filepath;
1212 }
1213 free(filepath);
1214 }
1215
1216
1217 /************************************************************************
1218 Return the location of given .ibd if it was previously read
1219 from .isl file.
1220 @return NULL or destination .ibd file path. */
1221 static
1222 const char *
tablespace_filepath(const char * ibd_filepath)1223 tablespace_filepath(const char *ibd_filepath)
1224 {
1225 std::map<std::string, std::string>::iterator it;
1226
1227 it = tablespace_locations.find(ibd_filepath);
1228
1229 if (it != tablespace_locations.end()) {
1230 return it->second.c_str();
1231 }
1232
1233 return NULL;
1234 }
1235
1236
1237 /************************************************************************
1238 Copy or move file depending on current mode.
1239 @return true in case of success. */
1240 static
1241 bool
copy_or_move_file(const char * src_file_path,const char * dst_file_path,const char * dst_dir,uint thread_n,bool copy=xtrabackup_copy_back)1242 copy_or_move_file(const char *src_file_path,
1243 const char *dst_file_path,
1244 const char *dst_dir,
1245 uint thread_n,
1246 bool copy = xtrabackup_copy_back)
1247 {
1248 ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */
1249 char filedir[FN_REFLEN];
1250 size_t filedir_len;
1251 bool ret;
1252
1253 /* read the link from .isl file */
1254 if (ends_with(src_file_path, ".isl")) {
1255 char *ibd_filepath;
1256
1257 ibd_filepath = strdup(src_file_path);
1258 strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd");
1259
1260 read_link_file(ibd_filepath, src_file_path);
1261
1262 free(ibd_filepath);
1263 }
1264
1265 /* check if there is .isl file */
1266 if (ends_with(src_file_path, ".ibd")) {
1267 char *link_filepath;
1268 const char *filepath;
1269
1270 link_filepath = strdup(src_file_path);
1271 strcpy(link_filepath + strlen(link_filepath) - 3, "isl");
1272
1273 read_link_file(src_file_path, link_filepath);
1274
1275 filepath = tablespace_filepath(src_file_path);
1276
1277 if (filepath != NULL) {
1278 dirname_part(filedir, filepath, &filedir_len);
1279
1280 dst_file_path = filepath + filedir_len;
1281 dst_dir = filedir;
1282
1283 if (!directory_exists(dst_dir, true)) {
1284 ret = false;
1285 free(link_filepath);
1286 goto cleanup;
1287 }
1288
1289 datasink = ds_create(dst_dir, DS_TYPE_LOCAL);
1290 }
1291
1292 free(link_filepath);
1293 }
1294
1295 ret = (copy ?
1296 copy_file(datasink, src_file_path, dst_file_path, thread_n) :
1297 move_file(datasink, src_file_path, dst_file_path,
1298 dst_dir, thread_n));
1299
1300 cleanup:
1301
1302 if (datasink != ds_data) {
1303 ds_destroy(datasink);
1304 }
1305
1306 return(ret);
1307 }
1308
1309
1310
1311
1312 static
1313 bool
backup_files(const char * from,bool prep_mode)1314 backup_files(const char *from, bool prep_mode)
1315 {
1316 char rsync_tmpfile_name[FN_REFLEN];
1317 FILE *rsync_tmpfile = NULL;
1318 datadir_iter_t *it;
1319 datadir_node_t node;
1320 bool ret = true;
1321
1322 if (prep_mode && !opt_rsync) {
1323 return(true);
1324 }
1325
1326 if (opt_rsync) {
1327 snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1328 "%s/%s%d", opt_mysql_tmpdir,
1329 "xtrabackup_rsyncfiles_pass",
1330 prep_mode ? 1 : 2);
1331 rsync_tmpfile = fopen(rsync_tmpfile_name, "w");
1332 if (rsync_tmpfile == NULL) {
1333 msg("Error: can't create file %s",
1334 rsync_tmpfile_name);
1335 return(false);
1336 }
1337 }
1338
1339 msg("Starting %s non-InnoDB tables and files",
1340 prep_mode ? "prep copy of" : "to backup");
1341
1342 datadir_node_init(&node);
1343 it = datadir_iter_new(from);
1344
1345 while (datadir_iter_next(it, &node)) {
1346
1347 if (!node.is_empty_dir) {
1348 if (opt_rsync) {
1349 ret = datafile_rsync_backup(node.filepath,
1350 !prep_mode, rsync_tmpfile);
1351 } else {
1352 ret = datafile_copy_backup(node.filepath, 1);
1353 }
1354 if (!ret) {
1355 msg("Failed to copy file %s", node.filepath);
1356 goto out;
1357 }
1358 } else if (!prep_mode) {
1359 /* backup fake file into empty directory */
1360 char path[FN_REFLEN];
1361 snprintf(path, sizeof(path),
1362 "%s/db.opt", node.filepath);
1363 if (!(ret = backup_file_printf(
1364 trim_dotslash(path), "%s", ""))) {
1365 msg("Failed to create file %s", path);
1366 goto out;
1367 }
1368 }
1369 }
1370
1371 if (opt_rsync) {
1372 std::stringstream cmd;
1373 int err;
1374
1375 if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1376 fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename);
1377 rsync_list.insert(buffer_pool_filename);
1378 }
1379 if (file_exists("ib_lru_dump")) {
1380 fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump");
1381 rsync_list.insert("ib_lru_dump");
1382 }
1383
1384 fclose(rsync_tmpfile);
1385 rsync_tmpfile = NULL;
1386
1387 cmd << "rsync -t . --files-from=" << rsync_tmpfile_name
1388 << " " << xtrabackup_target_dir;
1389
1390 msg("Starting rsync as: %s", cmd.str().c_str());
1391 if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) {
1392 msg("Error: rsync failed with error code %d", err);
1393 ret = false;
1394 goto out;
1395 }
1396 msg("rsync finished successfully.");
1397
1398 if (!prep_mode && !opt_no_lock) {
1399 char path[FN_REFLEN];
1400 char dst_path[FN_REFLEN];
1401 char *newline;
1402
1403 /* Remove files that have been removed between first and
1404 second passes. Cannot use "rsync --delete" because it
1405 does not work with --files-from. */
1406 snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1407 "%s/%s", opt_mysql_tmpdir,
1408 "xtrabackup_rsyncfiles_pass1");
1409
1410 rsync_tmpfile = fopen(rsync_tmpfile_name, "r");
1411 if (rsync_tmpfile == NULL) {
1412 msg("Error: can't open file %s",
1413 rsync_tmpfile_name);
1414 ret = false;
1415 goto out;
1416 }
1417
1418 while (fgets(path, sizeof(path), rsync_tmpfile)) {
1419
1420 newline = strchr(path, '\n');
1421 if (newline) {
1422 *newline = 0;
1423 }
1424 if (rsync_list.count(path) < 1) {
1425 snprintf(dst_path, sizeof(dst_path),
1426 "%s/%s", xtrabackup_target_dir,
1427 path);
1428 msg("Removing %s", dst_path);
1429 unlink(dst_path);
1430 }
1431 }
1432
1433 fclose(rsync_tmpfile);
1434 rsync_tmpfile = NULL;
1435 }
1436 }
1437
1438 msg("Finished %s non-InnoDB tables and files",
1439 prep_mode ? "a prep copy of" : "backing up");
1440
1441 out:
1442 datadir_iter_free(it);
1443 datadir_node_free(&node);
1444
1445 if (rsync_tmpfile != NULL) {
1446 fclose(rsync_tmpfile);
1447 }
1448
1449 return(ret);
1450 }
1451
1452 void backup_fix_ddl(CorruptedPages &);
1453
get_current_lsn(MYSQL * connection)1454 lsn_t get_current_lsn(MYSQL *connection)
1455 {
1456 static const char lsn_prefix[] = "\nLog sequence number ";
1457 lsn_t lsn = 0;
1458 if (MYSQL_RES *res = xb_mysql_query(connection,
1459 "SHOW ENGINE INNODB STATUS",
1460 true, false)) {
1461 if (MYSQL_ROW row = mysql_fetch_row(res)) {
1462 const char *p= strstr(row[2], lsn_prefix);
1463 DBUG_ASSERT(p);
1464 if (p) {
1465 p += sizeof lsn_prefix - 1;
1466 lsn = lsn_t(strtoll(p, NULL, 10));
1467 }
1468 }
1469 mysql_free_result(res);
1470 }
1471 return lsn;
1472 }
1473
1474 lsn_t server_lsn_after_lock;
1475 extern void backup_wait_for_lsn(lsn_t lsn);
1476 /** Start --backup */
backup_start(CorruptedPages & corrupted_pages)1477 bool backup_start(CorruptedPages &corrupted_pages)
1478 {
1479 if (!opt_no_lock) {
1480 if (opt_safe_slave_backup) {
1481 if (!wait_for_safe_slave(mysql_connection)) {
1482 return(false);
1483 }
1484 }
1485
1486 if (!backup_files(fil_path_to_mysql_datadir, true)) {
1487 return(false);
1488 }
1489
1490 history_lock_time = time(NULL);
1491
1492 if (!lock_tables(mysql_connection)) {
1493 return(false);
1494 }
1495 server_lsn_after_lock = get_current_lsn(mysql_connection);
1496 }
1497
1498 if (!backup_files(fil_path_to_mysql_datadir, false)) {
1499 return(false);
1500 }
1501
1502 if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) {
1503 return false;
1504 }
1505
1506 if (has_rocksdb_plugin()) {
1507 rocksdb_create_checkpoint();
1508 }
1509
1510 msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock);
1511 backup_wait_for_lsn(server_lsn_after_lock);
1512 backup_fix_ddl(corrupted_pages);
1513
1514 // There is no need to stop slave thread before coping non-Innodb data when
1515 // --no-lock option is used because --no-lock option requires that no DDL or
1516 // DML to non-transaction tables can occur.
1517 if (opt_no_lock) {
1518 if (opt_safe_slave_backup) {
1519 if (!wait_for_safe_slave(mysql_connection)) {
1520 return(false);
1521 }
1522 }
1523 }
1524
1525 if (opt_slave_info) {
1526 lock_binlog_maybe(mysql_connection);
1527
1528 if (!write_slave_info(mysql_connection)) {
1529 return(false);
1530 }
1531 }
1532
1533 /* The only reason why Galera/binlog info is written before
1534 wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup
1535 binary will start streamig a temporary copy of REDO log to stdout and
1536 thus, any streaming from innobackupex would interfere. The only way to
1537 avoid that is to have a single process, i.e. merge innobackupex and
1538 xtrabackup. */
1539 if (opt_galera_info) {
1540 if (!write_galera_info(mysql_connection)) {
1541 return(false);
1542 }
1543 write_current_binlog_file(mysql_connection);
1544 }
1545
1546 if (opt_binlog_info == BINLOG_INFO_ON) {
1547
1548 lock_binlog_maybe(mysql_connection);
1549 write_binlog_info(mysql_connection);
1550 }
1551
1552 if (have_flush_engine_logs && !opt_no_lock) {
1553 msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...");
1554 xb_mysql_query(mysql_connection,
1555 "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false);
1556 }
1557
1558 return(true);
1559 }
1560
1561 /** Release resources after backup_start() */
backup_release()1562 void backup_release()
1563 {
1564 /* release all locks */
1565 if (!opt_no_lock) {
1566 unlock_all(mysql_connection);
1567 history_lock_time = 0;
1568 } else {
1569 history_lock_time = time(NULL) - history_lock_time;
1570 }
1571
1572 if (opt_lock_ddl_per_table) {
1573 mdl_unlock_all();
1574 }
1575
1576 if (opt_safe_slave_backup && sql_thread_started) {
1577 msg("Starting slave SQL thread");
1578 xb_mysql_query(mysql_connection,
1579 "START SLAVE SQL_THREAD", false);
1580 }
1581 }
1582
1583 /** Finish after backup_start() and backup_release() */
backup_finish()1584 bool backup_finish()
1585 {
1586 /* Copy buffer pool dump or LRU dump */
1587 if (!opt_rsync) {
1588 if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1589 const char *dst_name;
1590
1591 dst_name = trim_dotslash(buffer_pool_filename);
1592 copy_file(ds_data, buffer_pool_filename, dst_name, 0);
1593 }
1594 if (file_exists("ib_lru_dump")) {
1595 copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0);
1596 }
1597 }
1598
1599 if (has_rocksdb_plugin()) {
1600 rocksdb_backup_checkpoint();
1601 }
1602
1603 msg("Backup created in directory '%s'", xtrabackup_target_dir);
1604 if (mysql_binlog_position != NULL) {
1605 msg("MySQL binlog position: %s", mysql_binlog_position);
1606 }
1607 if (mysql_slave_position && opt_slave_info) {
1608 msg("MySQL slave binlog position: %s",
1609 mysql_slave_position);
1610 }
1611
1612 if (!write_backup_config_file()) {
1613 return(false);
1614 }
1615
1616 if (!write_xtrabackup_info(mysql_connection, XTRABACKUP_INFO,
1617 opt_history != 0, true)) {
1618 return(false);
1619 }
1620
1621 return(true);
1622 }
1623
1624 bool
ibx_copy_incremental_over_full()1625 ibx_copy_incremental_over_full()
1626 {
1627 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
1628 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
1629 NULL};
1630 const char *sup_files[] = {"xtrabackup_binlog_info",
1631 "xtrabackup_galera_info",
1632 "xtrabackup_slave_info",
1633 "xtrabackup_info",
1634 "ib_lru_dump",
1635 NULL};
1636 datadir_iter_t *it = NULL;
1637 datadir_node_t node;
1638 bool ret = true;
1639 char path[FN_REFLEN];
1640 int i;
1641
1642 datadir_node_init(&node);
1643
1644 /* If we were applying an incremental change set, we need to make
1645 sure non-InnoDB files and xtrabackup_* metainfo files are copied
1646 to the full backup directory. */
1647
1648 if (xtrabackup_incremental) {
1649
1650 ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL);
1651
1652 it = datadir_iter_new(xtrabackup_incremental_dir);
1653
1654 while (datadir_iter_next(it, &node)) {
1655
1656 /* copy only non-innodb files */
1657
1658 if (node.is_empty_dir
1659 || !filename_matches(node.filepath, ext_list)) {
1660 continue;
1661 }
1662
1663 if (file_exists(node.filepath_rel)) {
1664 unlink(node.filepath_rel);
1665 }
1666
1667 if (!(ret = copy_file(ds_data, node.filepath,
1668 node.filepath_rel, 1))) {
1669 msg("Failed to copy file %s",
1670 node.filepath);
1671 goto cleanup;
1672 }
1673 }
1674
1675 if (!(ret = backup_files_from_datadir(xtrabackup_incremental_dir)))
1676 goto cleanup;
1677
1678 /* copy buffer pool dump */
1679 if (innobase_buffer_pool_filename) {
1680 const char *src_name;
1681
1682 src_name = trim_dotslash(innobase_buffer_pool_filename);
1683
1684 snprintf(path, sizeof(path), "%s/%s",
1685 xtrabackup_incremental_dir,
1686 src_name);
1687
1688 if (file_exists(path)) {
1689 copy_file(ds_data, path,
1690 innobase_buffer_pool_filename, 0);
1691 }
1692 }
1693
1694 /* copy supplementary files */
1695
1696 for (i = 0; sup_files[i]; i++) {
1697 snprintf(path, sizeof(path), "%s/%s",
1698 xtrabackup_incremental_dir,
1699 sup_files[i]);
1700
1701 if (file_exists(path))
1702 {
1703 if (file_exists(sup_files[i])) {
1704 unlink(sup_files[i]);
1705 }
1706 copy_file(ds_data, path, sup_files[i], 0);
1707 }
1708 }
1709
1710 if (directory_exists(ROCKSDB_BACKUP_DIR, false)) {
1711 if (my_rmtree(ROCKSDB_BACKUP_DIR, MYF(0))) {
1712 die("Can't remove " ROCKSDB_BACKUP_DIR);
1713 }
1714 }
1715 snprintf(path, sizeof(path), "%s/" ROCKSDB_BACKUP_DIR, xtrabackup_incremental_dir);
1716 if (directory_exists(path, false)) {
1717 if (my_mkdir(ROCKSDB_BACKUP_DIR, 0777, MYF(0))) {
1718 die("my_mkdir failed for " ROCKSDB_BACKUP_DIR);
1719 }
1720 copy_or_move_dir(path, ROCKSDB_BACKUP_DIR, true, true);
1721 }
1722 }
1723
1724
1725 cleanup:
1726 if (it != NULL) {
1727 datadir_iter_free(it);
1728 }
1729
1730 if (ds_data != NULL) {
1731 ds_destroy(ds_data);
1732 }
1733
1734 datadir_node_free(&node);
1735
1736 return(ret);
1737 }
1738
1739 bool
ibx_cleanup_full_backup()1740 ibx_cleanup_full_backup()
1741 {
1742 const char *ext_list[] = {"delta", "meta", "ibd", NULL};
1743 datadir_iter_t *it = NULL;
1744 datadir_node_t node;
1745 bool ret = true;
1746
1747 datadir_node_init(&node);
1748
1749 /* If we are applying an incremental change set, we need to make
1750 sure non-InnoDB files are cleaned up from full backup dir before
1751 we copy files from incremental dir. */
1752
1753 it = datadir_iter_new(xtrabackup_target_dir);
1754
1755 while (datadir_iter_next(it, &node)) {
1756
1757 if (node.is_empty_dir) {
1758 #ifdef _WIN32
1759 DeleteFile(node.filepath);
1760 #else
1761 rmdir(node.filepath);
1762 #endif
1763 }
1764
1765 if (xtrabackup_incremental && !node.is_empty_dir
1766 && !filename_matches(node.filepath, ext_list)) {
1767 unlink(node.filepath);
1768 }
1769 }
1770
1771 datadir_iter_free(it);
1772
1773 datadir_node_free(&node);
1774
1775 return(ret);
1776 }
1777
1778 bool
apply_log_finish()1779 apply_log_finish()
1780 {
1781 if (!ibx_cleanup_full_backup()
1782 || !ibx_copy_incremental_over_full()) {
1783 return(false);
1784 }
1785
1786 return(true);
1787 }
1788
1789 bool
copy_back()1790 copy_back()
1791 {
1792 bool ret = false;
1793 datadir_iter_t *it = NULL;
1794 datadir_node_t node;
1795 char *dst_dir;
1796
1797 memset(&node, 0, sizeof(node));
1798
1799 if (!opt_force_non_empty_dirs) {
1800 if (!directory_exists_and_empty(mysql_data_home,
1801 "Original data")) {
1802 return(false);
1803 }
1804 } else {
1805 if (!directory_exists(mysql_data_home, true)) {
1806 return(false);
1807 }
1808 }
1809 if (srv_undo_dir && *srv_undo_dir
1810 && !directory_exists(srv_undo_dir, true)) {
1811 return(false);
1812 }
1813 if (innobase_data_home_dir && *innobase_data_home_dir
1814 && !directory_exists(innobase_data_home_dir, true)) {
1815 return(false);
1816 }
1817 if (srv_log_group_home_dir && *srv_log_group_home_dir
1818 && !directory_exists(srv_log_group_home_dir, true)) {
1819 return(false);
1820 }
1821
1822 /* cd to backup directory */
1823 if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
1824 {
1825 msg("Can't my_setwd %s", xtrabackup_target_dir);
1826 return(false);
1827 }
1828
1829 /* parse data file path */
1830
1831 if (!innobase_data_file_path) {
1832 innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
1833 }
1834
1835 srv_sys_space.set_path(".");
1836
1837 if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
1838 msg("syntax error in innodb_data_file_path");
1839 return(false);
1840 }
1841
1842 srv_max_n_threads = 1000;
1843 sync_check_init();
1844 ut_crc32_init();
1845
1846 /* copy undo tablespaces */
1847
1848
1849 dst_dir = (srv_undo_dir && *srv_undo_dir)
1850 ? srv_undo_dir : mysql_data_home;
1851
1852 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1853
1854 for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) {
1855 char filename[20];
1856 sprintf(filename, "undo%03u", i);
1857 if (!file_exists(filename)) {
1858 break;
1859 }
1860 if (!(ret = copy_or_move_file(filename, filename,
1861 dst_dir, 1))) {
1862 goto cleanup;
1863 }
1864 }
1865
1866 ds_destroy(ds_data);
1867 ds_data = NULL;
1868
1869 /* copy redo logs */
1870
1871 dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir)
1872 ? srv_log_group_home_dir : mysql_data_home;
1873
1874 /* --backup generates a single ib_logfile0, which we must copy
1875 if it exists. */
1876
1877 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1878 MY_STAT stat_arg;
1879 if (!my_stat("ib_logfile0", &stat_arg, MYF(0)) || !stat_arg.st_size) {
1880 /* After completed --prepare, redo log files are redundant.
1881 We must delete any redo logs at the destination, so that
1882 the database will not jump to a different log sequence number
1883 (LSN). */
1884
1885 for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) {
1886 char filename[FN_REFLEN];
1887 snprintf(filename, sizeof filename, "%s/ib_logfile%u",
1888 dst_dir, i);
1889 unlink(filename);
1890 }
1891 } else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0",
1892 dst_dir, 1))) {
1893 goto cleanup;
1894 }
1895 ds_destroy(ds_data);
1896
1897 /* copy innodb system tablespace(s) */
1898
1899 dst_dir = (innobase_data_home_dir && *innobase_data_home_dir)
1900 ? innobase_data_home_dir : mysql_data_home;
1901
1902 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1903
1904 for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1905 end(srv_sys_space.end());
1906 iter != end;
1907 ++iter) {
1908 const char *filename = base_name(iter->name());
1909
1910 if (!(ret = copy_or_move_file(filename, iter->name(),
1911 dst_dir, 1))) {
1912 goto cleanup;
1913 }
1914 }
1915
1916 ds_destroy(ds_data);
1917
1918 /* copy the rest of tablespaces */
1919 ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL);
1920
1921 it = datadir_iter_new(".", false);
1922
1923 datadir_node_init(&node);
1924
1925 while (datadir_iter_next(it, &node)) {
1926 const char *ext_list[] = {"backup-my.cnf",
1927 "xtrabackup_binary", "xtrabackup_binlog_info",
1928 "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
1929 NULL};
1930 const char *filename;
1931 char c_tmp;
1932 int i_tmp;
1933 bool is_ibdata_file;
1934
1935 if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
1936 #ifdef _WIN32
1937 || strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
1938 #endif
1939 )
1940 {
1941 // copied at later step
1942 continue;
1943 }
1944
1945 /* create empty directories */
1946 if (node.is_empty_dir) {
1947 char path[FN_REFLEN];
1948
1949 snprintf(path, sizeof(path), "%s/%s",
1950 mysql_data_home, node.filepath_rel);
1951
1952 msg("Creating directory %s", path);
1953
1954 if (mkdirp(path, 0777, MYF(0)) < 0) {
1955 char errbuf[MYSYS_STRERROR_SIZE];
1956 my_strerror(errbuf, sizeof(errbuf), my_errno);
1957 msg("Can not create directory %s: %s",
1958 path, errbuf);
1959 ret = false;
1960
1961 goto cleanup;
1962
1963 }
1964
1965 msg(" ...done.");
1966
1967 continue;
1968 }
1969
1970 filename = base_name(node.filepath);
1971
1972 /* skip .qp files */
1973 if (filename_matches(filename, ext_list)) {
1974 continue;
1975 }
1976
1977 /* skip undo tablespaces */
1978 if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) {
1979 continue;
1980 }
1981
1982 /* skip the redo log (it was already copied) */
1983 if (!strcmp(filename, "ib_logfile0")) {
1984 continue;
1985 }
1986
1987 /* skip innodb data files */
1988 is_ibdata_file = false;
1989 for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1990 end(srv_sys_space.end()); iter != end; ++iter) {
1991 const char *ibfile = base_name(iter->name());
1992 if (strcmp(ibfile, filename) == 0) {
1993 is_ibdata_file = true;
1994 break;
1995 }
1996 }
1997 if (is_ibdata_file) {
1998 continue;
1999 }
2000
2001 if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel,
2002 mysql_data_home, 1))) {
2003 goto cleanup;
2004 }
2005 }
2006
2007 /* copy buffer pool dump */
2008
2009 if (innobase_buffer_pool_filename) {
2010 const char *src_name;
2011 char path[FN_REFLEN];
2012
2013 src_name = trim_dotslash(innobase_buffer_pool_filename);
2014
2015 snprintf(path, sizeof(path), "%s/%s",
2016 mysql_data_home,
2017 src_name);
2018
2019 /* could be already copied with other files
2020 from data directory */
2021 if (file_exists(src_name) &&
2022 !file_exists(innobase_buffer_pool_filename)) {
2023 copy_or_move_file(src_name,
2024 innobase_buffer_pool_filename,
2025 mysql_data_home, 0);
2026 }
2027 }
2028
2029 rocksdb_copy_back();
2030
2031 cleanup:
2032 if (it != NULL) {
2033 datadir_iter_free(it);
2034 }
2035
2036 datadir_node_free(&node);
2037
2038 if (ds_data != NULL) {
2039 ds_destroy(ds_data);
2040 }
2041
2042 ds_data = NULL;
2043
2044 sync_check_close();
2045 return(ret);
2046 }
2047
2048 bool
decrypt_decompress_file(const char * filepath,uint thread_n)2049 decrypt_decompress_file(const char *filepath, uint thread_n)
2050 {
2051 std::stringstream cmd, message;
2052 char *dest_filepath = strdup(filepath);
2053 bool needs_action = false;
2054
2055 cmd << IF_WIN("type ","cat ") << filepath;
2056
2057 if (opt_decompress
2058 && ends_with(filepath, ".qp")) {
2059 cmd << " | qpress -dio ";
2060 dest_filepath[strlen(dest_filepath) - 3] = 0;
2061 if (needs_action) {
2062 message << " and ";
2063 }
2064 message << "decompressing";
2065 needs_action = true;
2066 }
2067
2068 cmd << " > " << dest_filepath;
2069 message << " " << filepath;
2070
2071 free(dest_filepath);
2072
2073 if (needs_action) {
2074
2075 msg(thread_n,"%s\n", message.str().c_str());
2076
2077 if (system(cmd.str().c_str()) != 0) {
2078 return(false);
2079 }
2080
2081 if (opt_remove_original) {
2082 msg(thread_n, "Removing %s", filepath);
2083 if (my_delete(filepath, MYF(MY_WME)) != 0) {
2084 return(false);
2085 }
2086 }
2087 }
2088
2089 return(true);
2090 }
2091
2092 static
2093 os_thread_ret_t STDCALL
decrypt_decompress_thread_func(void * arg)2094 decrypt_decompress_thread_func(void *arg)
2095 {
2096 bool ret = true;
2097 datadir_node_t node;
2098 datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg);
2099
2100 datadir_node_init(&node);
2101
2102 while (datadir_iter_next(ctxt->it, &node)) {
2103
2104 /* skip empty directories in backup */
2105 if (node.is_empty_dir) {
2106 continue;
2107 }
2108
2109 if (!ends_with(node.filepath, ".qp")) {
2110 continue;
2111 }
2112
2113 if (!(ret = decrypt_decompress_file(node.filepath,
2114 ctxt->n_thread))) {
2115 goto cleanup;
2116 }
2117 }
2118
2119 cleanup:
2120
2121 datadir_node_free(&node);
2122
2123 pthread_mutex_lock(ctxt->count_mutex);
2124 --(*ctxt->count);
2125 pthread_mutex_unlock(ctxt->count_mutex);
2126
2127 ctxt->ret = ret;
2128
2129 os_thread_exit();
2130 OS_THREAD_DUMMY_RETURN;
2131 }
2132
2133 bool
decrypt_decompress()2134 decrypt_decompress()
2135 {
2136 bool ret;
2137 datadir_iter_t *it = NULL;
2138
2139 srv_max_n_threads = 1000;
2140 sync_check_init();
2141
2142 /* cd to backup directory */
2143 if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
2144 {
2145 msg("Can't my_setwd %s", xtrabackup_target_dir);
2146 return(false);
2147 }
2148
2149 /* copy the rest of tablespaces */
2150 ds_data = ds_create(".", DS_TYPE_LOCAL);
2151
2152 it = datadir_iter_new(".", false);
2153
2154 ut_a(xtrabackup_parallel >= 0);
2155
2156 ret = run_data_threads(it, decrypt_decompress_thread_func,
2157 xtrabackup_parallel ? xtrabackup_parallel : 1);
2158
2159 if (it != NULL) {
2160 datadir_iter_free(it);
2161 }
2162
2163 if (ds_data != NULL) {
2164 ds_destroy(ds_data);
2165 }
2166
2167 ds_data = NULL;
2168
2169 sync_check_close();
2170
2171 return(ret);
2172 }
2173
2174 /*
2175 Copy some files from top level datadir.
2176 Do not copy the Innodb files (ibdata1, redo log files),
2177 as this is done in a separate step.
2178 */
backup_files_from_datadir(const char * dir_path)2179 static bool backup_files_from_datadir(const char *dir_path)
2180 {
2181 os_file_dir_t dir = os_file_opendir(dir_path);
2182 if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false;
2183
2184 os_file_stat_t info;
2185 bool ret = true;
2186 while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
2187
2188 if (info.type != OS_FILE_TYPE_FILE)
2189 continue;
2190
2191 const char *pname = strrchr(info.name, OS_PATH_SEPARATOR);
2192 if (!pname)
2193 pname = info.name;
2194
2195 if (!starts_with(pname, "aws-kms-key") &&
2196 !starts_with(pname, "aria_log"))
2197 /* For ES exchange the above line with the following code:
2198 (!xtrabackup_prepare || !xtrabackup_incremental_dir ||
2199 !starts_with(pname, "aria_log")))
2200 */
2201 continue;
2202
2203 if (xtrabackup_prepare && xtrabackup_incremental_dir &&
2204 file_exists(info.name))
2205 unlink(info.name);
2206
2207 std::string full_path(dir_path);
2208 full_path.append(1, OS_PATH_SEPARATOR).append(info.name);
2209 if (!(ret = copy_file(ds_data, full_path.c_str() , info.name, 1)))
2210 break;
2211 }
2212 os_file_closedir(dir);
2213 return ret;
2214 }
2215
2216
rocksdb_remove_checkpoint_directory()2217 static int rocksdb_remove_checkpoint_directory()
2218 {
2219 xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
2220 return 0;
2221 }
2222
has_rocksdb_plugin()2223 static bool has_rocksdb_plugin()
2224 {
2225 static bool first_time = true;
2226 static bool has_plugin= false;
2227 if (!first_time || !xb_backup_rocksdb)
2228 return has_plugin;
2229
2230 const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
2231 MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
2232 MYSQL_ROW row = mysql_fetch_row(result);
2233 if (row)
2234 has_plugin = !strcmp(row[0], "1");
2235 mysql_free_result(result);
2236 first_time = false;
2237 return has_plugin;
2238 }
2239
trim_trailing_dir_sep(char * path)2240 static char *trim_trailing_dir_sep(char *path)
2241 {
2242 size_t path_len = strlen(path);
2243 while (path_len)
2244 {
2245 char c = path[path_len - 1];
2246 if (c == '/' IF_WIN(|| c == '\\', ))
2247 path_len--;
2248 else
2249 break;
2250 }
2251 path[path_len] = 0;
2252 return path;
2253 }
2254
2255 /*
2256 Create a file hardlink.
2257 @return true on success, false on error.
2258 */
make_hardlink(const char * from_path,const char * to_path)2259 static bool make_hardlink(const char *from_path, const char *to_path)
2260 {
2261 DBUG_EXECUTE_IF("no_hardlinks", return false;);
2262 char to_path_full[FN_REFLEN];
2263 if (!is_abs_path(to_path))
2264 {
2265 fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
2266 }
2267 else
2268 {
2269 strncpy(to_path_full, to_path, sizeof(to_path_full));
2270 }
2271 #ifdef _WIN32
2272 return CreateHardLink(to_path_full, from_path, NULL);
2273 #else
2274 return !link(from_path, to_path_full);
2275 #endif
2276 }
2277
2278 /*
2279 Copies or moves a directory (non-recursively so far).
2280 Helper function used to backup rocksdb checkpoint, or copy-back the
2281 rocksdb files.
2282
2283 Has optimization that allows to use hardlinks when possible
2284 (source and destination are directories on the same device)
2285 */
copy_or_move_dir(const char * from,const char * to,bool do_copy,bool allow_hardlinks)2286 static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
2287 {
2288 datadir_node_t node;
2289 datadir_node_init(&node);
2290 datadir_iter_t *it = datadir_iter_new(from, false);
2291
2292 while (datadir_iter_next(it, &node))
2293 {
2294 char to_path[FN_REFLEN];
2295 const char *from_path = node.filepath;
2296 snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
2297 bool rc = false;
2298 if (do_copy && allow_hardlinks)
2299 {
2300 rc = make_hardlink(from_path, to_path);
2301 if (rc)
2302 {
2303 msg("Creating hardlink from %s to %s",from_path, to_path);
2304 }
2305 else
2306 {
2307 allow_hardlinks = false;
2308 }
2309 }
2310
2311 if (!rc)
2312 {
2313 rc = (do_copy ?
2314 copy_file(ds_data, from_path, to_path, 1) :
2315 move_file(ds_data, from_path, node.filepath_rel,
2316 to, 1));
2317 }
2318 if (!rc)
2319 die("copy or move file failed");
2320 }
2321 datadir_iter_free(it);
2322 datadir_node_free(&node);
2323
2324 }
2325
2326 /*
2327 Obtain user level lock , to protect the checkpoint directory of the server
2328 from being user/overwritten by different backup processes, if backups are
2329 running in parallel.
2330
2331 This lock will be acquired before rocksdb checkpoint is created, held
2332 while all files from it are being copied to their final backup destination,
2333 and finally released after the checkpoint is removed.
2334 */
rocksdb_lock_checkpoint()2335 static void rocksdb_lock_checkpoint()
2336 {
2337 msg("Obtaining rocksdb checkpoint lock.");
2338 MYSQL_RES *res =
2339 xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);
2340
2341 MYSQL_ROW r = mysql_fetch_row(res);
2342 if (r && r[0] && strcmp(r[0], "1"))
2343 {
2344 msg("Could not obtain rocksdb checkpont lock.");
2345 exit(EXIT_FAILURE);
2346 }
2347 mysql_free_result(res);
2348 }
2349
rocksdb_unlock_checkpoint()2350 static void rocksdb_unlock_checkpoint()
2351 {
2352 xb_mysql_query(mysql_connection,
2353 "SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
2354 }
2355
2356
2357 /*
2358 Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
2359 directory.
2360 A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
2361 acquired be this function.
2362 */
2363 #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
2364 static char rocksdb_checkpoint_dir[FN_REFLEN];
2365
rocksdb_create_checkpoint()2366 static void rocksdb_create_checkpoint()
2367 {
2368 MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
2369 MYSQL_ROW row = mysql_fetch_row(result);
2370
2371 DBUG_ASSERT(row && row[0] && row[1]);
2372
2373 char *rocksdbdir = row[0];
2374 char *datadir = row[1];
2375
2376 if (is_abs_path(rocksdbdir))
2377 {
2378 snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
2379 "%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
2380 }
2381 else
2382 {
2383 snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
2384 "%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
2385 trim_dotslash(rocksdbdir));
2386 }
2387 mysql_free_result(result);
2388
2389 #ifdef _WIN32
2390 for (char *p = rocksdb_checkpoint_dir; *p; p++)
2391 if (*p == '\\') *p = '/';
2392 #endif
2393
2394 rocksdb_lock_checkpoint();
2395
2396 if (!access(rocksdb_checkpoint_dir, 0))
2397 {
2398 msg("Removing rocksdb checkpoint from previous backup attempt.");
2399 rocksdb_remove_checkpoint_directory();
2400 }
2401
2402 char query[FN_REFLEN + 32];
2403 snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
2404 xb_mysql_query(mysql_connection, query, false, true);
2405 }
2406
2407 /*
2408 Copy files from rocksdb temporary checkpoint to final destination.
2409 remove temp.checkpoint directory (in server's datadir)
2410 and release user level lock acquired inside rocksdb_create_checkpoint().
2411 */
rocksdb_backup_checkpoint()2412 static void rocksdb_backup_checkpoint()
2413 {
2414 msg("Backing up rocksdb files.");
2415 char rocksdb_backup_dir[FN_REFLEN];
2416 snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
2417 bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
2418 if (backup_to_directory)
2419 {
2420 if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
2421 die("Can't create rocksdb backup directory %s", rocksdb_backup_dir);
2422 }
2423 }
2424 copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
2425 rocksdb_remove_checkpoint_directory();
2426 rocksdb_unlock_checkpoint();
2427 }
2428
2429 /*
2430 Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
2431 */
rocksdb_copy_back()2432 static void rocksdb_copy_back() {
2433 if (access(ROCKSDB_BACKUP_DIR, 0))
2434 return;
2435 char rocksdb_home_dir[FN_REFLEN];
2436 if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
2437 strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof rocksdb_home_dir - 1);
2438 rocksdb_home_dir[sizeof rocksdb_home_dir - 1] = '\0';
2439 } else {
2440 snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home,
2441 xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
2442 }
2443 mkdirp(rocksdb_home_dir, 0777, MYF(0));
2444 copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
2445 }
2446