1 /******************************************************
2 hot backup tool for InnoDB
3 (c) 2009-2015 Percona LLC and/or its affiliates
4 (c) 2017 MariaDB
5 Originally Created 3/3/2009 Yasufumi Kinoshita
6 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
7 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; version 2 of the License.
12
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
21
22 *******************************************************
23
24 This file incorporates work covered by the following copyright and
25 permission notice:
26
27 Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
28
29 This program is free software; you can redistribute it and/or modify it under
30 the terms of the GNU General Public License as published by the Free Software
31 Foundation; version 2 of the License.
32
33 This program is distributed in the hope that it will be useful, but WITHOUT
34 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
35 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
36
37 You should have received a copy of the GNU General Public License along with
38 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
39 Street, Fifth Floor, Boston, MA 02110-1335 USA
40
41 *******************************************************/
42
43 #include <my_global.h>
44 #include <os0file.h>
45 #include <my_dir.h>
46 #include <ut0mem.h>
47 #include <srv0start.h>
48 #include <fil0fil.h>
49 #include <trx0sys.h>
50 #include <set>
51 #include <string>
52 #include <mysqld.h>
53 #include <sstream>
54 #include "fil_cur.h"
55 #include "xtrabackup.h"
56 #include "common.h"
57 #include "backup_copy.h"
58 #include "backup_mysql.h"
59 #include <btr0btr.h>
60
61 #define ROCKSDB_BACKUP_DIR "#rocksdb"
62
63 /* list of files to sync for --rsync mode */
64 static std::set<std::string> rsync_list;
65 /* locations of tablespaces read from .isl files */
66 static std::map<std::string, std::string> tablespace_locations;
67
68 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
69 bool binlog_locked;
70
71 static void rocksdb_create_checkpoint();
72 static bool has_rocksdb_plugin();
73 static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
74 static void rocksdb_backup_checkpoint();
75 static void rocksdb_copy_back();
76
is_abs_path(const char * path)77 static bool is_abs_path(const char *path)
78 {
79 #ifdef _WIN32
80 return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
81 #else
82 return path[0] == '/';
83 #endif
84 }
85
86 /************************************************************************
87 Struct represents file or directory. */
88 struct datadir_node_t {
89 ulint dbpath_len;
90 char *filepath;
91 ulint filepath_len;
92 char *filepath_rel;
93 ulint filepath_rel_len;
94 bool is_empty_dir;
95 bool is_file;
96 };
97
98 /************************************************************************
99 Holds the state needed to enumerate files in MySQL data directory. */
100 struct datadir_iter_t {
101 char *datadir_path;
102 char *dbpath;
103 ulint dbpath_len;
104 char *filepath;
105 ulint filepath_len;
106 char *filepath_rel;
107 ulint filepath_rel_len;
108 pthread_mutex_t mutex;
109 os_file_dir_t dir;
110 os_file_dir_t dbdir;
111 os_file_stat_t dbinfo;
112 os_file_stat_t fileinfo;
113 dberr_t err;
114 bool is_empty_dir;
115 bool is_file;
116 bool skip_first_level;
117 };
118
119
120 /************************************************************************
121 Represents the context of the thread processing MySQL data directory. */
122 struct datadir_thread_ctxt_t {
123 datadir_iter_t *it;
124 uint n_thread;
125 uint *count;
126 pthread_mutex_t* count_mutex;
127 os_thread_id_t id;
128 bool ret;
129 };
130
131 static bool backup_files_from_datadir(const char *dir_path);
132
133 /************************************************************************
134 Retirn true if character if file separator */
135 bool
is_path_separator(char c)136 is_path_separator(char c)
137 {
138 return(c == FN_LIBCHAR || c == FN_LIBCHAR2);
139 }
140
141
142 /************************************************************************
143 Fill the node struct. Memory for node need to be allocated and freed by
144 the caller. It is caller responsibility to initialize node with
145 datadir_node_init and cleanup the memory with datadir_node_free.
146 Node can not be shared between threads. */
147 static
148 void
datadir_node_fill(datadir_node_t * node,datadir_iter_t * it)149 datadir_node_fill(datadir_node_t *node, datadir_iter_t *it)
150 {
151 if (node->filepath_len < it->filepath_len) {
152 free(node->filepath);
153 node->filepath = (char*)(malloc(it->filepath_len));
154 node->filepath_len = it->filepath_len;
155 }
156 if (node->filepath_rel_len < it->filepath_rel_len) {
157 free(node->filepath_rel);
158 node->filepath_rel = (char*)(malloc(it->filepath_rel_len));
159 node->filepath_rel_len = it->filepath_rel_len;
160 }
161
162 strcpy(node->filepath, it->filepath);
163 strcpy(node->filepath_rel, it->filepath_rel);
164 node->is_empty_dir = it->is_empty_dir;
165 node->is_file = it->is_file;
166 }
167
168 static
169 void
datadir_node_free(datadir_node_t * node)170 datadir_node_free(datadir_node_t *node)
171 {
172 free(node->filepath);
173 free(node->filepath_rel);
174 memset(node, 0, sizeof(datadir_node_t));
175 }
176
177 static
178 void
datadir_node_init(datadir_node_t * node)179 datadir_node_init(datadir_node_t *node)
180 {
181 memset(node, 0, sizeof(datadir_node_t));
182 }
183
184
185 /************************************************************************
186 Create the MySQL data directory iterator. Memory needs to be released
187 with datadir_iter_free. Position should be advanced with
188 datadir_iter_next_file. Iterator can be shared between multiple
189 threads. It is guaranteed that each thread receives unique file from
190 data directory into its local node struct. */
191 static
192 datadir_iter_t *
datadir_iter_new(const char * path,bool skip_first_level=true)193 datadir_iter_new(const char *path, bool skip_first_level = true)
194 {
195 datadir_iter_t *it;
196
197 it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t)));
198 if (!it)
199 goto error;
200 memset(it, 0, sizeof(datadir_iter_t));
201
202 pthread_mutex_init(&it->mutex, NULL);
203 it->datadir_path = strdup(path);
204
205 it->dir = os_file_opendir(it->datadir_path);
206
207 if (it->dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
208
209 goto error;
210 }
211
212 it->err = DB_SUCCESS;
213
214 it->dbpath_len = FN_REFLEN;
215 it->dbpath = static_cast<char*>(malloc(it->dbpath_len));
216
217 it->filepath_len = FN_REFLEN;
218 it->filepath = static_cast<char*>(malloc(it->filepath_len));
219
220 it->filepath_rel_len = FN_REFLEN;
221 it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len));
222
223 it->skip_first_level = skip_first_level;
224
225 return(it);
226
227 error:
228 free(it);
229
230 return(NULL);
231 }
232
233 static
234 bool
datadir_iter_next_database(datadir_iter_t * it)235 datadir_iter_next_database(datadir_iter_t *it)
236 {
237 if (it->dbdir != NULL) {
238 if (os_file_closedir_failed(it->dbdir)) {
239 msg("Warning: could not"
240 " close database directory %s", it->dbpath);
241 it->err = DB_ERROR;
242
243 }
244 it->dbdir = NULL;
245 }
246
247 while (os_file_readdir_next_file(it->datadir_path,
248 it->dir, &it->dbinfo) == 0) {
249 ulint len;
250
251 if ((it->dbinfo.type == OS_FILE_TYPE_FILE
252 && it->skip_first_level)
253 || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
254
255 continue;
256 }
257
258 /* We found a symlink or a directory; try opening it to see
259 if a symlink is a directory */
260
261 len = strlen(it->datadir_path)
262 + strlen (it->dbinfo.name) + 2;
263 if (len > it->dbpath_len) {
264 it->dbpath_len = len;
265 free(it->dbpath);
266
267 it->dbpath = static_cast<char*>(
268 malloc(it->dbpath_len));
269 }
270 snprintf(it->dbpath, it->dbpath_len, "%s/%s",
271 it->datadir_path, it->dbinfo.name);
272 os_normalize_path(it->dbpath);
273
274 if (it->dbinfo.type == OS_FILE_TYPE_FILE) {
275 it->is_file = true;
276 return(true);
277 }
278
279 if (check_if_skip_database_by_path(it->dbpath)) {
280 msg("Skipping db: %s", it->dbpath);
281 continue;
282 }
283
284 /* We want wrong directory permissions to be a fatal error for
285 XtraBackup. */
286 it->dbdir = os_file_opendir(it->dbpath);
287
288 if (it->dir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
289 it->is_file = false;
290 return(true);
291 }
292
293 }
294
295 return(false);
296 }
297
298 /************************************************************************
299 Concatenate n parts into single path */
300 static
301 void
make_path_n(int n,char ** path,ulint * path_len,...)302 make_path_n(int n, char **path, ulint *path_len, ...)
303 {
304 ulint len_needed = n + 1;
305 char *p;
306 int i;
307 va_list vl;
308
309 ut_ad(n > 0);
310
311 va_start(vl, path_len);
312 for (i = 0; i < n; i++) {
313 p = va_arg(vl, char*);
314 len_needed += strlen(p);
315 }
316 va_end(vl);
317
318 if (len_needed < *path_len) {
319 free(*path);
320 *path = static_cast<char*>(malloc(len_needed));
321 }
322
323 va_start(vl, path_len);
324 p = va_arg(vl, char*);
325 strcpy(*path, p);
326 for (i = 1; i < n; i++) {
327 size_t plen;
328 p = va_arg(vl, char*);
329 plen = strlen(*path);
330 if (!is_path_separator((*path)[plen - 1])) {
331 (*path)[plen] = FN_LIBCHAR;
332 (*path)[plen + 1] = 0;
333 }
334 strcat(*path + plen, p);
335 }
336 va_end(vl);
337 }
338
339 static
340 bool
datadir_iter_next_file(datadir_iter_t * it)341 datadir_iter_next_file(datadir_iter_t *it)
342 {
343 if (it->is_file && it->dbpath) {
344 make_path_n(2, &it->filepath, &it->filepath_len,
345 it->datadir_path, it->dbinfo.name);
346
347 make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
348 it->dbinfo.name);
349
350 it->is_empty_dir = false;
351 it->is_file = false;
352
353 return(true);
354 }
355
356 if (!it->dbpath || !it->dbdir) {
357
358 return(false);
359 }
360
361 while (os_file_readdir_next_file(it->dbpath, it->dbdir,
362 &it->fileinfo) == 0) {
363
364 if (it->fileinfo.type == OS_FILE_TYPE_DIR) {
365
366 continue;
367 }
368
369 /* We found a symlink or a file */
370 make_path_n(3, &it->filepath, &it->filepath_len,
371 it->datadir_path, it->dbinfo.name,
372 it->fileinfo.name);
373
374 make_path_n(2, &it->filepath_rel, &it->filepath_rel_len,
375 it->dbinfo.name, it->fileinfo.name);
376
377 it->is_empty_dir = false;
378
379 return(true);
380 }
381
382 return(false);
383 }
384
385 static
386 bool
datadir_iter_next(datadir_iter_t * it,datadir_node_t * node)387 datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
388 {
389 bool ret = true;
390
391 pthread_mutex_lock(&it->mutex);
392
393 if (datadir_iter_next_file(it)) {
394
395 datadir_node_fill(node, it);
396
397 goto done;
398 }
399
400 while (datadir_iter_next_database(it)) {
401
402 if (datadir_iter_next_file(it)) {
403
404 datadir_node_fill(node, it);
405
406 goto done;
407 }
408
409 make_path_n(2, &it->filepath, &it->filepath_len,
410 it->datadir_path, it->dbinfo.name);
411
412 make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
413 it->dbinfo.name);
414
415 it->is_empty_dir = true;
416
417 datadir_node_fill(node, it);
418
419 goto done;
420 }
421
422 /* nothing found */
423 ret = false;
424
425 done:
426 pthread_mutex_unlock(&it->mutex);
427
428 return(ret);
429 }
430
431 /************************************************************************
432 Interface to read MySQL data file sequentially. One should open file
433 with datafile_open to get cursor and close the cursor with
434 datafile_close. Cursor can not be shared between multiple
435 threads. */
436 static
437 void
datadir_iter_free(datadir_iter_t * it)438 datadir_iter_free(datadir_iter_t *it)
439 {
440 pthread_mutex_destroy(&it->mutex);
441
442 if (it->dbdir) {
443
444 os_file_closedir(it->dbdir);
445 }
446
447 if (it->dir) {
448
449 os_file_closedir(it->dir);
450 }
451
452 free(it->dbpath);
453 free(it->filepath);
454 free(it->filepath_rel);
455 free(it->datadir_path);
456 free(it);
457 }
458
459
460 /************************************************************************
461 Holds the state needed to copy single data file. */
462 struct datafile_cur_t {
463 pfs_os_file_t file;
464 char rel_path[FN_REFLEN];
465 char abs_path[FN_REFLEN];
466 MY_STAT statinfo;
467 uint thread_n;
468 byte* buf;
469 size_t buf_size;
470 size_t buf_read;
471 size_t buf_offset;
472
datafile_cur_tdatafile_cur_t473 explicit datafile_cur_t(const char* filename = NULL) :
474 file(), thread_n(0), buf(NULL), buf_size(0),
475 buf_read(0), buf_offset(0)
476 {
477 memset(rel_path, 0, sizeof rel_path);
478 if (filename) {
479 strncpy(abs_path, filename, sizeof abs_path - 1);
480 abs_path[(sizeof abs_path) - 1] = 0;
481 } else {
482 abs_path[0] = '\0';
483 }
484 rel_path[0] = '\0';
485 memset(&statinfo, 0, sizeof statinfo);
486 }
487 };
488
489 static
490 void
datafile_close(datafile_cur_t * cursor)491 datafile_close(datafile_cur_t *cursor)
492 {
493 if (cursor->file != OS_FILE_CLOSED) {
494 os_file_close(cursor->file);
495 }
496 free(cursor->buf);
497 }
498
499 static
500 bool
datafile_open(const char * file,datafile_cur_t * cursor,uint thread_n)501 datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
502 {
503 bool success;
504
505 new (cursor) datafile_cur_t(file);
506
507 /* Get the relative path for the destination tablespace name, i.e. the
508 one that can be appended to the backup root directory. Non-system
509 tablespaces may have absolute paths for remote tablespaces in MySQL
510 5.6+. We want to make "local" copies for the backup. */
511 strncpy(cursor->rel_path,
512 xb_get_relative_path(cursor->abs_path, FALSE),
513 (sizeof cursor->rel_path) - 1);
514 cursor->rel_path[(sizeof cursor->rel_path) - 1] = '\0';
515
516 cursor->file = os_file_create_simple_no_error_handling(
517 0, cursor->abs_path,
518 OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success);
519 if (!success) {
520 /* The following call prints an error message */
521 os_file_get_last_error(TRUE);
522
523 msg(thread_n,"error: cannot open "
524 "file %s", cursor->abs_path);
525
526 return(false);
527 }
528
529 if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) {
530 msg(thread_n, "error: cannot stat %s", cursor->abs_path);
531 datafile_close(cursor);
532 return(false);
533 }
534
535 posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
536
537 cursor->buf_size = 10 * 1024 * 1024;
538 cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size));
539
540 return(true);
541 }
542
543
544 static
545 xb_fil_cur_result_t
datafile_read(datafile_cur_t * cursor)546 datafile_read(datafile_cur_t *cursor)
547 {
548 ulint to_read;
549
550 xtrabackup_io_throttling();
551
552 to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset,
553 cursor->buf_size);
554
555 if (to_read == 0) {
556 return(XB_FIL_CUR_EOF);
557 }
558
559 if (os_file_read(IORequestRead,
560 cursor->file, cursor->buf, cursor->buf_offset,
561 to_read) != DB_SUCCESS) {
562 return(XB_FIL_CUR_ERROR);
563 }
564
565 posix_fadvise(cursor->file, cursor->buf_offset, to_read,
566 POSIX_FADV_DONTNEED);
567
568 cursor->buf_read = to_read;
569 cursor->buf_offset += to_read;
570
571 return(XB_FIL_CUR_SUCCESS);
572 }
573
574
575
576 /************************************************************************
577 Check to see if a file exists.
578 Takes name of the file to check.
579 @return true if file exists. */
580 static
581 bool
file_exists(const char * filename)582 file_exists(const char *filename)
583 {
584 MY_STAT stat_arg;
585
586 if (!my_stat(filename, &stat_arg, MYF(0))) {
587
588 return(false);
589 }
590
591 return(true);
592 }
593
594 /************************************************************************
595 Trim leading slashes from absolute path so it becomes relative */
596 static
597 const char *
trim_dotslash(const char * path)598 trim_dotslash(const char *path)
599 {
600 while (*path) {
601 if (is_path_separator(*path)) {
602 ++path;
603 continue;
604 }
605 if (*path == '.' && is_path_separator(path[1])) {
606 path += 2;
607 continue;
608 }
609 break;
610 }
611
612 return(path);
613 }
614
615
616
617 /************************************************************************
618 Check if string ends with given suffix.
619 @return true if string ends with given suffix. */
620 bool
ends_with(const char * str,const char * suffix)621 ends_with(const char *str, const char *suffix)
622 {
623 size_t suffix_len = strlen(suffix);
624 size_t str_len = strlen(str);
625 return(str_len >= suffix_len
626 && strcmp(str + str_len - suffix_len, suffix) == 0);
627 }
628
starts_with(const char * str,const char * prefix)629 static bool starts_with(const char *str, const char *prefix)
630 {
631 return strncmp(str, prefix, strlen(prefix)) == 0;
632 }
633
634 /************************************************************************
635 Create directories recursively.
636 @return 0 if directories created successfully. */
637 static
638 int
mkdirp(const char * pathname,int Flags,myf MyFlags)639 mkdirp(const char *pathname, int Flags, myf MyFlags)
640 {
641 char *parent, *p;
642
643 /* make a parent directory path */
644 if (!(parent= strdup(pathname)))
645 return(-1);
646
647 for (p = parent + strlen(parent);
648 !is_path_separator(*p) && p != parent; p--) ;
649
650 *p = 0;
651
652 /* try to make parent directory */
653 if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) {
654 free(parent);
655 return(-1);
656 }
657
658 /* make this one if parent has been made */
659 if (my_mkdir(pathname, Flags, MyFlags) == 0) {
660 free(parent);
661 return(0);
662 }
663
664 /* if it already exists that is fine */
665 if (errno == EEXIST) {
666 free(parent);
667 return(0);
668 }
669
670 free(parent);
671 return(-1);
672 }
673
674 /************************************************************************
675 Return true if first and second arguments are the same path. */
676 bool
equal_paths(const char * first,const char * second)677 equal_paths(const char *first, const char *second)
678 {
679 #ifdef HAVE_REALPATH
680 char *real_first, *real_second;
681 int result;
682
683 real_first = realpath(first, 0);
684 if (real_first == NULL) {
685 return false;
686 }
687
688 real_second = realpath(second, 0);
689 if (real_second == NULL) {
690 free(real_first);
691 return false;
692 }
693
694 result = strcmp(real_first, real_second);
695 free(real_first);
696 free(real_second);
697 return result == 0;
698 #else
699 return strcmp(first, second) == 0;
700 #endif
701 }
702
703 /************************************************************************
704 Check if directory exists. Optionally create directory if doesn't
705 exist.
706 @return true if directory exists and if it was created successfully. */
707 bool
directory_exists(const char * dir,bool create)708 directory_exists(const char *dir, bool create)
709 {
710 os_file_dir_t os_dir;
711 MY_STAT stat_arg;
712 char errbuf[MYSYS_STRERROR_SIZE];
713
714 if (my_stat(dir, &stat_arg, MYF(0)) == NULL) {
715
716 if (!create) {
717 return(false);
718 }
719
720 if (mkdirp(dir, 0777, MYF(0)) < 0) {
721 my_strerror(errbuf, sizeof(errbuf), my_errno);
722 msg("Can not create directory %s: %s", dir, errbuf);
723 return(false);
724 }
725 }
726
727 /* could be symlink */
728 os_dir = os_file_opendir(dir);
729
730 if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
731 my_strerror(errbuf, sizeof(errbuf), my_errno);
732 msg("Can not open directory %s: %s", dir,
733 errbuf);
734
735 return(false);
736 }
737
738 os_file_closedir(os_dir);
739
740 return(true);
741 }
742
743 /************************************************************************
744 Check that directory exists and it is empty. */
745 static
746 bool
directory_exists_and_empty(const char * dir,const char * comment)747 directory_exists_and_empty(const char *dir, const char *comment)
748 {
749 os_file_dir_t os_dir;
750 dberr_t err;
751 os_file_stat_t info;
752 bool empty;
753
754 if (!directory_exists(dir, true)) {
755 return(false);
756 }
757
758 os_dir = os_file_opendir(dir);
759
760 if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
761 msg("%s can not open directory %s", comment, dir);
762 return(false);
763 }
764
765 empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0);
766
767 os_file_closedir(os_dir);
768
769 if (!empty) {
770 msg("%s directory %s is not empty!", comment, dir);
771 }
772
773 return(empty);
774 }
775
776
777 /************************************************************************
778 Check if file name ends with given set of suffixes.
779 @return true if it does. */
780 static
781 bool
filename_matches(const char * filename,const char ** ext_list)782 filename_matches(const char *filename, const char **ext_list)
783 {
784 const char **ext;
785
786 for (ext = ext_list; *ext; ext++) {
787 if (ends_with(filename, *ext)) {
788 return(true);
789 }
790 }
791
792 return(false);
793 }
794
795
796 /************************************************************************
797 Copy data file for backup. Also check if it is allowed to copy by
798 comparing its name to the list of known data file types and checking
799 if passes the rules for partial backup.
800 @return true if file backed up or skipped successfully. */
801 static
802 bool
datafile_copy_backup(const char * filepath,uint thread_n)803 datafile_copy_backup(const char *filepath, uint thread_n)
804 {
805 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
806 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
807 NULL};
808
809 /* Get the name and the path for the tablespace. node->name always
810 contains the path (which may be absolute for remote tablespaces in
811 5.6+). space->name contains the tablespace name in the form
812 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
813 multi-node shared tablespace, space->name contains the name of the first
814 node, but that's irrelevant, since we only need node_name to match them
815 against filters, and the shared tablespace is always copied regardless
816 of the filters value. */
817
818 if (check_if_skip_table(filepath)) {
819 msg(thread_n,"Skipping %s.", filepath);
820 return(true);
821 }
822
823 if (filename_matches(filepath, ext_list)) {
824 return copy_file(ds_data, filepath, filepath, thread_n);
825 }
826
827 return(true);
828 }
829
830
831 /************************************************************************
832 Same as datafile_copy_backup, but put file name into the list for
833 rsync command. */
834 static
835 bool
datafile_rsync_backup(const char * filepath,bool save_to_list,FILE * f)836 datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
837 {
838 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
839 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
840 NULL};
841
842 /* Get the name and the path for the tablespace. node->name always
843 contains the path (which may be absolute for remote tablespaces in
844 5.6+). space->name contains the tablespace name in the form
845 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
846 multi-node shared tablespace, space->name contains the name of the first
847 node, but that's irrelevant, since we only need node_name to match them
848 against filters, and the shared tablespace is always copied regardless
849 of the filters value. */
850
851 if (check_if_skip_table(filepath)) {
852 return(true);
853 }
854
855 if (filename_matches(filepath, ext_list)) {
856 fprintf(f, "%s\n", filepath);
857 if (save_to_list) {
858 rsync_list.insert(filepath);
859 }
860 }
861
862 return(true);
863 }
864
backup_file_print_buf(const char * filename,const char * buf,int buf_len)865 bool backup_file_print_buf(const char *filename, const char *buf, int buf_len)
866 {
867 ds_file_t *dstfile = NULL;
868 MY_STAT stat; /* unused for now */
869 const char *action;
870
871 memset(&stat, 0, sizeof(stat));
872
873 stat.st_size = buf_len;
874 stat.st_mtime = my_time(0);
875
876 dstfile = ds_open(ds_data, filename, &stat);
877 if (dstfile == NULL) {
878 msg("error: Can't open the destination stream for %s",
879 filename);
880 goto error;
881 }
882
883 action = xb_get_copy_action("Writing");
884 msg("%s %s", action, filename);
885
886 if (buf_len == -1) {
887 goto error;
888 }
889
890 if (ds_write(dstfile, buf, buf_len)) {
891 goto error;
892 }
893
894 /* close */
895 msg(" ...done");
896
897 if (ds_close(dstfile)) {
898 goto error_close;
899 }
900
901 return(true);
902
903 error:
904 if (dstfile != NULL) {
905 ds_close(dstfile);
906 }
907
908 error_close:
909 msg("Error: backup file failed.");
910 return(false); /*ERROR*/
911
912 return true;
913 };
914
915 static
916 bool
backup_file_vprintf(const char * filename,const char * fmt,va_list ap)917 backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
918 {
919 char *buf = 0;
920 int buf_len;
921 buf_len = vasprintf(&buf, fmt, ap);
922 bool result = backup_file_print_buf(filename, buf, buf_len);
923 free(buf);
924 return result;
925 }
926
927 bool
backup_file_printf(const char * filename,const char * fmt,...)928 backup_file_printf(const char *filename, const char *fmt, ...)
929 {
930 bool result;
931 va_list ap;
932
933 va_start(ap, fmt);
934
935 result = backup_file_vprintf(filename, fmt, ap);
936
937 va_end(ap);
938
939 return(result);
940 }
941
942 static
943 bool
run_data_threads(datadir_iter_t * it,os_thread_func_t func,uint n)944 run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
945 {
946 datadir_thread_ctxt_t *data_threads;
947 uint i, count;
948 pthread_mutex_t count_mutex;
949 bool ret;
950
951 data_threads = (datadir_thread_ctxt_t*)
952 malloc(sizeof(datadir_thread_ctxt_t) * n);
953
954 pthread_mutex_init(&count_mutex, NULL);
955 count = n;
956
957 for (i = 0; i < n; i++) {
958 data_threads[i].it = it;
959 data_threads[i].n_thread = i + 1;
960 data_threads[i].count = &count;
961 data_threads[i].count_mutex = &count_mutex;
962 data_threads[i].id = os_thread_create(func, data_threads + i);
963 }
964
965 /* Wait for threads to exit */
966 while (1) {
967 os_thread_sleep(100000);
968 pthread_mutex_lock(&count_mutex);
969 if (count == 0) {
970 pthread_mutex_unlock(&count_mutex);
971 break;
972 }
973 pthread_mutex_unlock(&count_mutex);
974 }
975
976 pthread_mutex_destroy(&count_mutex);
977
978 ret = true;
979 for (i = 0; i < n; i++) {
980 ret = data_threads[i].ret && ret;
981 if (!data_threads[i].ret) {
982 msg("Error: thread %u failed.", i);
983 }
984 }
985
986 free(data_threads);
987
988 return(ret);
989 }
990
991 #ifdef _WIN32
992 #include <windows.h>
993 #include <accctrl.h>
994 #include <aclapi.h>
995 /*
996 On Windows, fix permission of the file after "copyback"
997 We assume that after copyback, mysqld will run as service as NetworkService
998 user, thus well give full permission on given file to that user.
999 */
1000
fix_win_file_permissions(const char * file)1001 static int fix_win_file_permissions(const char *file)
1002 {
1003 struct {
1004 TOKEN_USER tokenUser;
1005 BYTE buffer[SECURITY_MAX_SID_SIZE];
1006 } tokenInfoBuffer;
1007 HANDLE hFile = CreateFile(file, READ_CONTROL | WRITE_DAC, 0, NULL, OPEN_EXISTING,
1008 FILE_FLAG_BACKUP_SEMANTICS, NULL);
1009 if (hFile == INVALID_HANDLE_VALUE)
1010 return -1;
1011 ACL* pOldDACL;
1012 SECURITY_DESCRIPTOR* pSD = NULL;
1013 EXPLICIT_ACCESS ea = { 0 };
1014 PSID pSid = NULL;
1015
1016 GetSecurityInfo(hFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL,
1017 &pOldDACL, NULL, (void**)&pSD);
1018 DWORD size = SECURITY_MAX_SID_SIZE;
1019 pSid = (PSID)tokenInfoBuffer.buffer;
1020 if (!CreateWellKnownSid(WinNetworkServiceSid, NULL, pSid,
1021 &size))
1022 {
1023 return 1;
1024 }
1025 ea.Trustee.TrusteeForm = TRUSTEE_IS_SID;
1026 ea.Trustee.ptstrName = (LPTSTR)pSid;
1027
1028 ea.grfAccessMode = GRANT_ACCESS;
1029 ea.grfAccessPermissions = GENERIC_ALL;
1030 ea.grfInheritance = CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE;
1031 ea.Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
1032 ACL* pNewDACL = 0;
1033 DWORD err = SetEntriesInAcl(1, &ea, pOldDACL, &pNewDACL);
1034 if (!err)
1035 {
1036 DBUG_ASSERT(pNewDACL);
1037 SetSecurityInfo(hFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL,
1038 pNewDACL, NULL);
1039 LocalFree((HLOCAL)pNewDACL);
1040 }
1041 if (pSD != NULL)
1042 LocalFree((HLOCAL)pSD);
1043 CloseHandle(hFile);
1044 return 0;
1045 }
1046
1047 #endif
1048
1049
1050 /************************************************************************
1051 Copy file for backup/restore.
1052 @return true in case of success. */
1053 bool
copy_file(ds_ctxt_t * datasink,const char * src_file_path,const char * dst_file_path,uint thread_n)1054 copy_file(ds_ctxt_t *datasink,
1055 const char *src_file_path,
1056 const char *dst_file_path,
1057 uint thread_n)
1058 {
1059 char dst_name[FN_REFLEN];
1060 ds_file_t *dstfile = NULL;
1061 datafile_cur_t cursor;
1062 xb_fil_cur_result_t res;
1063 DBUG_ASSERT(datasink->datasink->remove);
1064 const char *dst_path =
1065 (xtrabackup_copy_back || xtrabackup_move_back)?
1066 dst_file_path : trim_dotslash(dst_file_path);
1067
1068 if (!datafile_open(src_file_path, &cursor, thread_n)) {
1069 goto error_close;
1070 }
1071
1072 strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
1073
1074 dstfile = ds_open(datasink, dst_path, &cursor.statinfo);
1075 if (dstfile == NULL) {
1076 msg(thread_n,"error: "
1077 "cannot open the destination stream for %s", dst_name);
1078 goto error;
1079 }
1080
1081 msg(thread_n, "%s %s to %s", xb_get_copy_action(), src_file_path, dstfile->path);
1082
1083 /* The main copy loop */
1084 while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
1085
1086 if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
1087 goto error;
1088 }
1089 DBUG_EXECUTE_IF("copy_file_error", errno=ENOSPC;goto error;);
1090 }
1091
1092 if (res == XB_FIL_CUR_ERROR) {
1093 goto error;
1094 }
1095
1096 /* close */
1097 msg(thread_n," ...done");
1098 datafile_close(&cursor);
1099 #ifdef _WIN32
1100 if (xtrabackup_copy_back || xtrabackup_move_back)
1101 ut_a(!fix_win_file_permissions(dstfile->path));
1102 #endif
1103 if (ds_close(dstfile)) {
1104 goto error_close;
1105 }
1106 return(true);
1107
1108 error:
1109 datafile_close(&cursor);
1110 if (dstfile != NULL) {
1111 datasink->datasink->remove(dstfile->path);
1112 ds_close(dstfile);
1113 }
1114
1115 error_close:
1116 msg(thread_n,"Error: copy_file() failed.");
1117 return(false); /*ERROR*/
1118 }
1119
1120
1121 /************************************************************************
1122 Try to move file by renaming it. If source and destination are on
1123 different devices fall back to copy and unlink.
1124 @return true in case of success. */
1125 static
1126 bool
move_file(ds_ctxt_t * datasink,const char * src_file_path,const char * dst_file_path,const char * dst_dir,uint thread_n)1127 move_file(ds_ctxt_t *datasink,
1128 const char *src_file_path,
1129 const char *dst_file_path,
1130 const char *dst_dir, uint thread_n)
1131 {
1132 char errbuf[MYSYS_STRERROR_SIZE];
1133 char dst_file_path_abs[FN_REFLEN];
1134 char dst_dir_abs[FN_REFLEN];
1135 size_t dirname_length;
1136
1137 snprintf(dst_file_path_abs, sizeof(dst_file_path_abs),
1138 "%s/%s", dst_dir, dst_file_path);
1139
1140 dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length);
1141
1142 if (!directory_exists(dst_dir_abs, true)) {
1143 return(false);
1144 }
1145
1146 if (file_exists(dst_file_path_abs)) {
1147 msg("Error: Move file %s to %s failed: Destination "
1148 "file exists", src_file_path, dst_file_path_abs);
1149 return(false);
1150 }
1151
1152 msg(thread_n,"Moving %s to %s", src_file_path, dst_file_path_abs);
1153
1154 if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
1155 if (my_errno == EXDEV) {
1156 /* Fallback to copy/unlink */
1157 if(!copy_file(datasink, src_file_path,
1158 dst_file_path, thread_n))
1159 return false;
1160 msg(thread_n,"Removing %s", src_file_path);
1161 if (unlink(src_file_path) != 0) {
1162 my_strerror(errbuf, sizeof(errbuf), errno);
1163 msg("Warning: unlink %s failed: %s",
1164 src_file_path,
1165 errbuf);
1166 }
1167 return true;
1168 }
1169 my_strerror(errbuf, sizeof(errbuf), my_errno);
1170 msg("Can not move file %s to %s: %s",
1171 src_file_path, dst_file_path_abs,
1172 errbuf);
1173 return(false);
1174 }
1175 #ifdef _WIN32
1176 if (xtrabackup_copy_back || xtrabackup_move_back)
1177 ut_a(!fix_win_file_permissions(dst_file_path_abs));
1178 #endif
1179 msg(thread_n," ...done");
1180
1181 return(true);
1182 }
1183
1184
1185 /************************************************************************
1186 Read link from .isl file if any and store it in the global map associated
1187 with given tablespace. */
1188 static
1189 void
read_link_file(const char * ibd_filepath,const char * link_filepath)1190 read_link_file(const char *ibd_filepath, const char *link_filepath)
1191 {
1192 char *filepath= NULL;
1193
1194 FILE *file = fopen(link_filepath, "r+b");
1195 if (file) {
1196 filepath = static_cast<char*>(malloc(OS_FILE_MAX_PATH));
1197
1198 os_file_read_string(file, filepath, OS_FILE_MAX_PATH);
1199 fclose(file);
1200
1201 if (strlen(filepath)) {
1202 /* Trim whitespace from end of filepath */
1203 ulint lastch = strlen(filepath) - 1;
1204 while (lastch > 4 && filepath[lastch] <= 0x20) {
1205 filepath[lastch--] = 0x00;
1206 }
1207 os_normalize_path(filepath);
1208 }
1209
1210 tablespace_locations[ibd_filepath] = filepath;
1211 }
1212 free(filepath);
1213 }
1214
1215
1216 /************************************************************************
1217 Return the location of given .ibd if it was previously read
1218 from .isl file.
1219 @return NULL or destination .ibd file path. */
1220 static
1221 const char *
tablespace_filepath(const char * ibd_filepath)1222 tablespace_filepath(const char *ibd_filepath)
1223 {
1224 std::map<std::string, std::string>::iterator it;
1225
1226 it = tablespace_locations.find(ibd_filepath);
1227
1228 if (it != tablespace_locations.end()) {
1229 return it->second.c_str();
1230 }
1231
1232 return NULL;
1233 }
1234
1235
1236 /************************************************************************
1237 Copy or move file depending on current mode.
1238 @return true in case of success. */
1239 static
1240 bool
copy_or_move_file(const char * src_file_path,const char * dst_file_path,const char * dst_dir,uint thread_n,bool copy=xtrabackup_copy_back)1241 copy_or_move_file(const char *src_file_path,
1242 const char *dst_file_path,
1243 const char *dst_dir,
1244 uint thread_n,
1245 bool copy = xtrabackup_copy_back)
1246 {
1247 ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */
1248 char filedir[FN_REFLEN];
1249 size_t filedir_len;
1250 bool ret;
1251
1252 /* read the link from .isl file */
1253 if (ends_with(src_file_path, ".isl")) {
1254 char *ibd_filepath;
1255
1256 ibd_filepath = strdup(src_file_path);
1257 strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd");
1258
1259 read_link_file(ibd_filepath, src_file_path);
1260
1261 free(ibd_filepath);
1262 }
1263
1264 /* check if there is .isl file */
1265 if (ends_with(src_file_path, ".ibd")) {
1266 char *link_filepath;
1267 const char *filepath;
1268
1269 link_filepath = strdup(src_file_path);
1270 strcpy(link_filepath + strlen(link_filepath) - 3, "isl");
1271
1272 read_link_file(src_file_path, link_filepath);
1273
1274 filepath = tablespace_filepath(src_file_path);
1275
1276 if (filepath != NULL) {
1277 dirname_part(filedir, filepath, &filedir_len);
1278
1279 dst_file_path = filepath + filedir_len;
1280 dst_dir = filedir;
1281
1282 if (!directory_exists(dst_dir, true)) {
1283 ret = false;
1284 free(link_filepath);
1285 goto cleanup;
1286 }
1287
1288 datasink = ds_create(dst_dir, DS_TYPE_LOCAL);
1289 }
1290
1291 free(link_filepath);
1292 }
1293
1294 ret = (copy ?
1295 copy_file(datasink, src_file_path, dst_file_path, thread_n) :
1296 move_file(datasink, src_file_path, dst_file_path,
1297 dst_dir, thread_n));
1298
1299 cleanup:
1300
1301 if (datasink != ds_data) {
1302 ds_destroy(datasink);
1303 }
1304
1305 return(ret);
1306 }
1307
1308
1309
1310
1311 static
1312 bool
backup_files(const char * from,bool prep_mode)1313 backup_files(const char *from, bool prep_mode)
1314 {
1315 char rsync_tmpfile_name[FN_REFLEN];
1316 FILE *rsync_tmpfile = NULL;
1317 datadir_iter_t *it;
1318 datadir_node_t node;
1319 bool ret = true;
1320
1321 if (prep_mode && !opt_rsync) {
1322 return(true);
1323 }
1324
1325 if (opt_rsync) {
1326 snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1327 "%s/%s%d", opt_mysql_tmpdir,
1328 "xtrabackup_rsyncfiles_pass",
1329 prep_mode ? 1 : 2);
1330 rsync_tmpfile = fopen(rsync_tmpfile_name, "w");
1331 if (rsync_tmpfile == NULL) {
1332 msg("Error: can't create file %s",
1333 rsync_tmpfile_name);
1334 return(false);
1335 }
1336 }
1337
1338 msg("Starting %s non-InnoDB tables and files",
1339 prep_mode ? "prep copy of" : "to backup");
1340
1341 datadir_node_init(&node);
1342 it = datadir_iter_new(from);
1343
1344 while (datadir_iter_next(it, &node)) {
1345
1346 if (!node.is_empty_dir) {
1347 if (opt_rsync) {
1348 ret = datafile_rsync_backup(node.filepath,
1349 !prep_mode, rsync_tmpfile);
1350 } else {
1351 ret = datafile_copy_backup(node.filepath, 1);
1352 }
1353 if (!ret) {
1354 msg("Failed to copy file %s", node.filepath);
1355 goto out;
1356 }
1357 } else if (!prep_mode) {
1358 /* backup fake file into empty directory */
1359 char path[FN_REFLEN];
1360 snprintf(path, sizeof(path),
1361 "%s/db.opt", node.filepath);
1362 if (!(ret = backup_file_printf(
1363 trim_dotslash(path), "%s", ""))) {
1364 msg("Failed to create file %s", path);
1365 goto out;
1366 }
1367 }
1368 }
1369
1370 if (opt_rsync) {
1371 std::stringstream cmd;
1372 int err;
1373
1374 if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1375 fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename);
1376 rsync_list.insert(buffer_pool_filename);
1377 }
1378 if (file_exists("ib_lru_dump")) {
1379 fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump");
1380 rsync_list.insert("ib_lru_dump");
1381 }
1382
1383 fclose(rsync_tmpfile);
1384 rsync_tmpfile = NULL;
1385
1386 cmd << "rsync -t . --files-from=" << rsync_tmpfile_name
1387 << " " << xtrabackup_target_dir;
1388
1389 msg("Starting rsync as: %s", cmd.str().c_str());
1390 if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) {
1391 msg("Error: rsync failed with error code %d", err);
1392 ret = false;
1393 goto out;
1394 }
1395 msg("rsync finished successfully.");
1396
1397 if (!prep_mode && !opt_no_lock) {
1398 char path[FN_REFLEN];
1399 char dst_path[FN_REFLEN];
1400 char *newline;
1401
1402 /* Remove files that have been removed between first and
1403 second passes. Cannot use "rsync --delete" because it
1404 does not work with --files-from. */
1405 snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1406 "%s/%s", opt_mysql_tmpdir,
1407 "xtrabackup_rsyncfiles_pass1");
1408
1409 rsync_tmpfile = fopen(rsync_tmpfile_name, "r");
1410 if (rsync_tmpfile == NULL) {
1411 msg("Error: can't open file %s",
1412 rsync_tmpfile_name);
1413 ret = false;
1414 goto out;
1415 }
1416
1417 while (fgets(path, sizeof(path), rsync_tmpfile)) {
1418
1419 newline = strchr(path, '\n');
1420 if (newline) {
1421 *newline = 0;
1422 }
1423 if (rsync_list.count(path) < 1) {
1424 snprintf(dst_path, sizeof(dst_path),
1425 "%s/%s", xtrabackup_target_dir,
1426 path);
1427 msg("Removing %s", dst_path);
1428 unlink(dst_path);
1429 }
1430 }
1431
1432 fclose(rsync_tmpfile);
1433 rsync_tmpfile = NULL;
1434 }
1435 }
1436
1437 msg("Finished %s non-InnoDB tables and files",
1438 prep_mode ? "a prep copy of" : "backing up");
1439
1440 out:
1441 datadir_iter_free(it);
1442 datadir_node_free(&node);
1443
1444 if (rsync_tmpfile != NULL) {
1445 fclose(rsync_tmpfile);
1446 }
1447
1448 return(ret);
1449 }
1450
1451 void backup_fix_ddl(CorruptedPages &);
1452
get_current_lsn(MYSQL * connection)1453 lsn_t get_current_lsn(MYSQL *connection)
1454 {
1455 static const char lsn_prefix[] = "\nLog sequence number ";
1456 lsn_t lsn = 0;
1457 if (MYSQL_RES *res = xb_mysql_query(connection,
1458 "SHOW ENGINE INNODB STATUS",
1459 true, false)) {
1460 if (MYSQL_ROW row = mysql_fetch_row(res)) {
1461 const char *p= strstr(row[2], lsn_prefix);
1462 DBUG_ASSERT(p);
1463 if (p) {
1464 p += sizeof lsn_prefix - 1;
1465 lsn = lsn_t(strtoll(p, NULL, 10));
1466 }
1467 }
1468 mysql_free_result(res);
1469 }
1470 return lsn;
1471 }
1472
1473 lsn_t server_lsn_after_lock;
1474 extern void backup_wait_for_lsn(lsn_t lsn);
1475 /** Start --backup */
backup_start(CorruptedPages & corrupted_pages)1476 bool backup_start(CorruptedPages &corrupted_pages)
1477 {
1478 if (!opt_no_lock) {
1479 if (opt_safe_slave_backup) {
1480 if (!wait_for_safe_slave(mysql_connection)) {
1481 return(false);
1482 }
1483 }
1484
1485 if (!backup_files(fil_path_to_mysql_datadir, true)) {
1486 return(false);
1487 }
1488
1489 history_lock_time = time(NULL);
1490
1491 if (!lock_tables(mysql_connection)) {
1492 return(false);
1493 }
1494 server_lsn_after_lock = get_current_lsn(mysql_connection);
1495 }
1496
1497 if (!backup_files(fil_path_to_mysql_datadir, false)) {
1498 return(false);
1499 }
1500
1501 if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) {
1502 return false;
1503 }
1504
1505 if (has_rocksdb_plugin()) {
1506 rocksdb_create_checkpoint();
1507 }
1508
1509 msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock);
1510 backup_wait_for_lsn(server_lsn_after_lock);
1511 backup_fix_ddl(corrupted_pages);
1512
1513 // There is no need to stop slave thread before coping non-Innodb data when
1514 // --no-lock option is used because --no-lock option requires that no DDL or
1515 // DML to non-transaction tables can occur.
1516 if (opt_no_lock) {
1517 if (opt_safe_slave_backup) {
1518 if (!wait_for_safe_slave(mysql_connection)) {
1519 return(false);
1520 }
1521 }
1522 }
1523
1524 if (opt_slave_info) {
1525 lock_binlog_maybe(mysql_connection);
1526
1527 if (!write_slave_info(mysql_connection)) {
1528 return(false);
1529 }
1530 }
1531
1532 /* The only reason why Galera/binlog info is written before
1533 wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup
1534 binary will start streamig a temporary copy of REDO log to stdout and
1535 thus, any streaming from innobackupex would interfere. The only way to
1536 avoid that is to have a single process, i.e. merge innobackupex and
1537 xtrabackup. */
1538 if (opt_galera_info) {
1539 if (!write_galera_info(mysql_connection)) {
1540 return(false);
1541 }
1542 write_current_binlog_file(mysql_connection);
1543 }
1544
1545 if (opt_binlog_info == BINLOG_INFO_ON) {
1546
1547 lock_binlog_maybe(mysql_connection);
1548 write_binlog_info(mysql_connection);
1549 }
1550
1551 if (have_flush_engine_logs && !opt_no_lock) {
1552 msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...");
1553 xb_mysql_query(mysql_connection,
1554 "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false);
1555 }
1556
1557 return(true);
1558 }
1559
1560 /** Release resources after backup_start() */
backup_release()1561 void backup_release()
1562 {
1563 /* release all locks */
1564 if (!opt_no_lock) {
1565 unlock_all(mysql_connection);
1566 history_lock_time = 0;
1567 } else {
1568 history_lock_time = time(NULL) - history_lock_time;
1569 }
1570
1571 if (opt_lock_ddl_per_table) {
1572 mdl_unlock_all();
1573 }
1574
1575 if (opt_safe_slave_backup && sql_thread_started) {
1576 msg("Starting slave SQL thread");
1577 xb_mysql_query(mysql_connection,
1578 "START SLAVE SQL_THREAD", false);
1579 }
1580 }
1581
1582 /** Finish after backup_start() and backup_release() */
backup_finish()1583 bool backup_finish()
1584 {
1585 /* Copy buffer pool dump or LRU dump */
1586 if (!opt_rsync) {
1587 if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1588 const char *dst_name;
1589
1590 dst_name = trim_dotslash(buffer_pool_filename);
1591 copy_file(ds_data, buffer_pool_filename, dst_name, 0);
1592 }
1593 if (file_exists("ib_lru_dump")) {
1594 copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0);
1595 }
1596 }
1597
1598 if (has_rocksdb_plugin()) {
1599 rocksdb_backup_checkpoint();
1600 }
1601
1602 msg("Backup created in directory '%s'", xtrabackup_target_dir);
1603 if (mysql_binlog_position != NULL) {
1604 msg("MySQL binlog position: %s", mysql_binlog_position);
1605 }
1606 if (mysql_slave_position && opt_slave_info) {
1607 msg("MySQL slave binlog position: %s",
1608 mysql_slave_position);
1609 }
1610
1611 if (!write_backup_config_file()) {
1612 return(false);
1613 }
1614
1615 if (!write_xtrabackup_info(mysql_connection, XTRABACKUP_INFO,
1616 opt_history != 0, true)) {
1617 return(false);
1618 }
1619
1620 return(true);
1621 }
1622
1623 bool
ibx_copy_incremental_over_full()1624 ibx_copy_incremental_over_full()
1625 {
1626 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
1627 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
1628 NULL};
1629 const char *sup_files[] = {"xtrabackup_binlog_info",
1630 "xtrabackup_galera_info",
1631 "xtrabackup_slave_info",
1632 "xtrabackup_info",
1633 "ib_lru_dump",
1634 NULL};
1635 datadir_iter_t *it = NULL;
1636 datadir_node_t node;
1637 bool ret = true;
1638 char path[FN_REFLEN];
1639 int i;
1640
1641 datadir_node_init(&node);
1642
1643 /* If we were applying an incremental change set, we need to make
1644 sure non-InnoDB files and xtrabackup_* metainfo files are copied
1645 to the full backup directory. */
1646
1647 if (xtrabackup_incremental) {
1648
1649 ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL);
1650
1651 it = datadir_iter_new(xtrabackup_incremental_dir);
1652
1653 while (datadir_iter_next(it, &node)) {
1654
1655 /* copy only non-innodb files */
1656
1657 if (node.is_empty_dir
1658 || !filename_matches(node.filepath, ext_list)) {
1659 continue;
1660 }
1661
1662 if (file_exists(node.filepath_rel)) {
1663 unlink(node.filepath_rel);
1664 }
1665
1666 if (!(ret = copy_file(ds_data, node.filepath,
1667 node.filepath_rel, 1))) {
1668 msg("Failed to copy file %s",
1669 node.filepath);
1670 goto cleanup;
1671 }
1672 }
1673
1674 if (!(ret = backup_files_from_datadir(xtrabackup_incremental_dir)))
1675 goto cleanup;
1676
1677 /* copy buffer pool dump */
1678 if (innobase_buffer_pool_filename) {
1679 const char *src_name;
1680
1681 src_name = trim_dotslash(innobase_buffer_pool_filename);
1682
1683 snprintf(path, sizeof(path), "%s/%s",
1684 xtrabackup_incremental_dir,
1685 src_name);
1686
1687 if (file_exists(path)) {
1688 copy_file(ds_data, path,
1689 innobase_buffer_pool_filename, 0);
1690 }
1691 }
1692
1693 /* copy supplementary files */
1694
1695 for (i = 0; sup_files[i]; i++) {
1696 snprintf(path, sizeof(path), "%s/%s",
1697 xtrabackup_incremental_dir,
1698 sup_files[i]);
1699
1700 if (file_exists(path))
1701 {
1702 if (file_exists(sup_files[i])) {
1703 unlink(sup_files[i]);
1704 }
1705 copy_file(ds_data, path, sup_files[i], 0);
1706 }
1707 }
1708
1709 if (directory_exists(ROCKSDB_BACKUP_DIR, false)) {
1710 if (my_rmtree(ROCKSDB_BACKUP_DIR, MYF(0))) {
1711 die("Can't remove " ROCKSDB_BACKUP_DIR);
1712 }
1713 }
1714 snprintf(path, sizeof(path), "%s/" ROCKSDB_BACKUP_DIR, xtrabackup_incremental_dir);
1715 if (directory_exists(path, false)) {
1716 if (my_mkdir(ROCKSDB_BACKUP_DIR, 0777, MYF(0))) {
1717 die("my_mkdir failed for " ROCKSDB_BACKUP_DIR);
1718 }
1719 copy_or_move_dir(path, ROCKSDB_BACKUP_DIR, true, true);
1720 }
1721 }
1722
1723
1724 cleanup:
1725 if (it != NULL) {
1726 datadir_iter_free(it);
1727 }
1728
1729 if (ds_data != NULL) {
1730 ds_destroy(ds_data);
1731 }
1732
1733 datadir_node_free(&node);
1734
1735 return(ret);
1736 }
1737
1738 bool
ibx_cleanup_full_backup()1739 ibx_cleanup_full_backup()
1740 {
1741 const char *ext_list[] = {"delta", "meta", "ibd", NULL};
1742 datadir_iter_t *it = NULL;
1743 datadir_node_t node;
1744 bool ret = true;
1745
1746 datadir_node_init(&node);
1747
1748 /* If we are applying an incremental change set, we need to make
1749 sure non-InnoDB files are cleaned up from full backup dir before
1750 we copy files from incremental dir. */
1751
1752 it = datadir_iter_new(xtrabackup_target_dir);
1753
1754 while (datadir_iter_next(it, &node)) {
1755
1756 if (node.is_empty_dir) {
1757 #ifdef _WIN32
1758 DeleteFile(node.filepath);
1759 #else
1760 rmdir(node.filepath);
1761 #endif
1762 }
1763
1764 if (xtrabackup_incremental && !node.is_empty_dir
1765 && !filename_matches(node.filepath, ext_list)) {
1766 unlink(node.filepath);
1767 }
1768 }
1769
1770 datadir_iter_free(it);
1771
1772 datadir_node_free(&node);
1773
1774 return(ret);
1775 }
1776
1777 bool
apply_log_finish()1778 apply_log_finish()
1779 {
1780 if (!ibx_cleanup_full_backup()
1781 || !ibx_copy_incremental_over_full()) {
1782 return(false);
1783 }
1784
1785 return(true);
1786 }
1787
1788 bool
copy_back()1789 copy_back()
1790 {
1791 bool ret = false;
1792 datadir_iter_t *it = NULL;
1793 datadir_node_t node;
1794 char *dst_dir;
1795
1796 memset(&node, 0, sizeof(node));
1797
1798 if (!opt_force_non_empty_dirs) {
1799 if (!directory_exists_and_empty(mysql_data_home,
1800 "Original data")) {
1801 return(false);
1802 }
1803 } else {
1804 if (!directory_exists(mysql_data_home, true)) {
1805 return(false);
1806 }
1807 }
1808 if (srv_undo_dir && *srv_undo_dir
1809 && !directory_exists(srv_undo_dir, true)) {
1810 return(false);
1811 }
1812 if (innobase_data_home_dir && *innobase_data_home_dir
1813 && !directory_exists(innobase_data_home_dir, true)) {
1814 return(false);
1815 }
1816 if (srv_log_group_home_dir && *srv_log_group_home_dir
1817 && !directory_exists(srv_log_group_home_dir, true)) {
1818 return(false);
1819 }
1820
1821 /* cd to backup directory */
1822 if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
1823 {
1824 msg("Can't my_setwd %s", xtrabackup_target_dir);
1825 return(false);
1826 }
1827
1828 /* parse data file path */
1829
1830 if (!innobase_data_file_path) {
1831 innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
1832 }
1833
1834 srv_sys_space.set_path(".");
1835
1836 if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
1837 msg("syntax error in innodb_data_file_path");
1838 return(false);
1839 }
1840
1841 srv_max_n_threads = 1000;
1842 sync_check_init();
1843
1844 /* copy undo tablespaces */
1845
1846
1847 dst_dir = (srv_undo_dir && *srv_undo_dir)
1848 ? srv_undo_dir : mysql_data_home;
1849
1850 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1851
1852 for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) {
1853 char filename[20];
1854 sprintf(filename, "undo%03u", i);
1855 if (!file_exists(filename)) {
1856 break;
1857 }
1858 if (!(ret = copy_or_move_file(filename, filename,
1859 dst_dir, 1))) {
1860 goto cleanup;
1861 }
1862 }
1863
1864 ds_destroy(ds_data);
1865 ds_data = NULL;
1866
1867 /* copy redo logs */
1868
1869 dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir)
1870 ? srv_log_group_home_dir : mysql_data_home;
1871
1872 /* --backup generates a single LOG_FILE_NAME, which we must copy
1873 if it exists. */
1874
1875 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1876 MY_STAT stat_arg;
1877 if (!my_stat(LOG_FILE_NAME, &stat_arg, MYF(0)) || !stat_arg.st_size) {
1878 /* After completed --prepare, redo log files are redundant.
1879 We must delete any redo logs at the destination, so that
1880 the database will not jump to a different log sequence number
1881 (LSN). */
1882
1883 char filename[FN_REFLEN];
1884 snprintf(filename, sizeof filename, "%s/%s0", dst_dir,
1885 LOG_FILE_NAME_PREFIX);
1886 unlink(filename);
1887 snprintf(filename, sizeof filename, "%s/%s101", dst_dir,
1888 LOG_FILE_NAME_PREFIX);
1889 unlink(filename);
1890 } else if (!(ret = copy_or_move_file(LOG_FILE_NAME, LOG_FILE_NAME,
1891 dst_dir, 1))) {
1892 goto cleanup;
1893 }
1894 ds_destroy(ds_data);
1895
1896 /* copy innodb system tablespace(s) */
1897
1898 dst_dir = (innobase_data_home_dir && *innobase_data_home_dir)
1899 ? innobase_data_home_dir : mysql_data_home;
1900
1901 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1902
1903 for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1904 end(srv_sys_space.end());
1905 iter != end;
1906 ++iter) {
1907 const char *filename = base_name(iter->name());
1908
1909 if (!(ret = copy_or_move_file(filename, iter->name(),
1910 dst_dir, 1))) {
1911 goto cleanup;
1912 }
1913 }
1914
1915 ds_destroy(ds_data);
1916
1917 /* copy the rest of tablespaces */
1918 ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL);
1919
1920 it = datadir_iter_new(".", false);
1921
1922 datadir_node_init(&node);
1923
1924 while (datadir_iter_next(it, &node)) {
1925 const char *ext_list[] = {"backup-my.cnf",
1926 "xtrabackup_binary", "xtrabackup_binlog_info",
1927 "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
1928 NULL};
1929 const char *filename;
1930 char c_tmp;
1931 int i_tmp;
1932 bool is_ibdata_file;
1933
1934 if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
1935 #ifdef _WIN32
1936 || strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
1937 #endif
1938 )
1939 {
1940 // copied at later step
1941 continue;
1942 }
1943
1944 /* create empty directories */
1945 if (node.is_empty_dir) {
1946 char path[FN_REFLEN];
1947
1948 snprintf(path, sizeof(path), "%s/%s",
1949 mysql_data_home, node.filepath_rel);
1950
1951 msg("Creating directory %s", path);
1952
1953 if (mkdirp(path, 0777, MYF(0)) < 0) {
1954 char errbuf[MYSYS_STRERROR_SIZE];
1955 my_strerror(errbuf, sizeof(errbuf), my_errno);
1956 msg("Can not create directory %s: %s",
1957 path, errbuf);
1958 ret = false;
1959
1960 goto cleanup;
1961
1962 }
1963
1964 msg(" ...done.");
1965
1966 continue;
1967 }
1968
1969 filename = base_name(node.filepath);
1970
1971 /* skip .qp files */
1972 if (filename_matches(filename, ext_list)) {
1973 continue;
1974 }
1975
1976 /* skip undo tablespaces */
1977 if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) {
1978 continue;
1979 }
1980
1981 /* skip the redo log (it was already copied) */
1982 if (!strcmp(filename, LOG_FILE_NAME)) {
1983 continue;
1984 }
1985
1986 /* skip innodb data files */
1987 is_ibdata_file = false;
1988 for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1989 end(srv_sys_space.end()); iter != end; ++iter) {
1990 const char *ibfile = base_name(iter->name());
1991 if (strcmp(ibfile, filename) == 0) {
1992 is_ibdata_file = true;
1993 break;
1994 }
1995 }
1996 if (is_ibdata_file) {
1997 continue;
1998 }
1999
2000 if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel,
2001 mysql_data_home, 1))) {
2002 goto cleanup;
2003 }
2004 }
2005
2006 /* copy buffer pool dump */
2007
2008 if (innobase_buffer_pool_filename) {
2009 const char *src_name;
2010 char path[FN_REFLEN];
2011
2012 src_name = trim_dotslash(innobase_buffer_pool_filename);
2013
2014 snprintf(path, sizeof(path), "%s/%s",
2015 mysql_data_home,
2016 src_name);
2017
2018 /* could be already copied with other files
2019 from data directory */
2020 if (file_exists(src_name) &&
2021 !file_exists(innobase_buffer_pool_filename)) {
2022 copy_or_move_file(src_name,
2023 innobase_buffer_pool_filename,
2024 mysql_data_home, 0);
2025 }
2026 }
2027
2028 rocksdb_copy_back();
2029
2030 cleanup:
2031 if (it != NULL) {
2032 datadir_iter_free(it);
2033 }
2034
2035 datadir_node_free(&node);
2036
2037 if (ds_data != NULL) {
2038 ds_destroy(ds_data);
2039 }
2040
2041 ds_data = NULL;
2042
2043 sync_check_close();
2044 return(ret);
2045 }
2046
2047 bool
decrypt_decompress_file(const char * filepath,uint thread_n)2048 decrypt_decompress_file(const char *filepath, uint thread_n)
2049 {
2050 std::stringstream cmd, message;
2051 char *dest_filepath = strdup(filepath);
2052 bool needs_action = false;
2053
2054 cmd << IF_WIN("type ","cat ") << filepath;
2055
2056 if (opt_decompress
2057 && ends_with(filepath, ".qp")) {
2058 cmd << " | qpress -dio ";
2059 dest_filepath[strlen(dest_filepath) - 3] = 0;
2060 if (needs_action) {
2061 message << " and ";
2062 }
2063 message << "decompressing";
2064 needs_action = true;
2065 }
2066
2067 cmd << " > " << dest_filepath;
2068 message << " " << filepath;
2069
2070 free(dest_filepath);
2071
2072 if (needs_action) {
2073
2074 msg(thread_n,"%s\n", message.str().c_str());
2075
2076 if (system(cmd.str().c_str()) != 0) {
2077 return(false);
2078 }
2079
2080 if (opt_remove_original) {
2081 msg(thread_n, "Removing %s", filepath);
2082 if (my_delete(filepath, MYF(MY_WME)) != 0) {
2083 return(false);
2084 }
2085 }
2086 }
2087
2088 return(true);
2089 }
2090
2091 static
2092 os_thread_ret_t STDCALL
decrypt_decompress_thread_func(void * arg)2093 decrypt_decompress_thread_func(void *arg)
2094 {
2095 bool ret = true;
2096 datadir_node_t node;
2097 datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg);
2098
2099 datadir_node_init(&node);
2100
2101 while (datadir_iter_next(ctxt->it, &node)) {
2102
2103 /* skip empty directories in backup */
2104 if (node.is_empty_dir) {
2105 continue;
2106 }
2107
2108 if (!ends_with(node.filepath, ".qp")) {
2109 continue;
2110 }
2111
2112 if (!(ret = decrypt_decompress_file(node.filepath,
2113 ctxt->n_thread))) {
2114 goto cleanup;
2115 }
2116 }
2117
2118 cleanup:
2119
2120 datadir_node_free(&node);
2121
2122 pthread_mutex_lock(ctxt->count_mutex);
2123 --(*ctxt->count);
2124 pthread_mutex_unlock(ctxt->count_mutex);
2125
2126 ctxt->ret = ret;
2127
2128 os_thread_exit();
2129 OS_THREAD_DUMMY_RETURN;
2130 }
2131
2132 bool
decrypt_decompress()2133 decrypt_decompress()
2134 {
2135 bool ret;
2136 datadir_iter_t *it = NULL;
2137
2138 srv_max_n_threads = 1000;
2139 sync_check_init();
2140
2141 /* cd to backup directory */
2142 if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
2143 {
2144 msg("Can't my_setwd %s", xtrabackup_target_dir);
2145 return(false);
2146 }
2147
2148 /* copy the rest of tablespaces */
2149 ds_data = ds_create(".", DS_TYPE_LOCAL);
2150
2151 it = datadir_iter_new(".", false);
2152
2153 ut_a(xtrabackup_parallel >= 0);
2154
2155 ret = run_data_threads(it, decrypt_decompress_thread_func,
2156 xtrabackup_parallel ? xtrabackup_parallel : 1);
2157
2158 if (it != NULL) {
2159 datadir_iter_free(it);
2160 }
2161
2162 if (ds_data != NULL) {
2163 ds_destroy(ds_data);
2164 }
2165
2166 ds_data = NULL;
2167
2168 sync_check_close();
2169
2170 return(ret);
2171 }
2172
2173 /*
2174 Copy some files from top level datadir.
2175 Do not copy the Innodb files (ibdata1, redo log files),
2176 as this is done in a separate step.
2177 */
backup_files_from_datadir(const char * dir_path)2178 static bool backup_files_from_datadir(const char *dir_path)
2179 {
2180 os_file_dir_t dir = os_file_opendir(dir_path);
2181 if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false;
2182
2183 os_file_stat_t info;
2184 bool ret = true;
2185 while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
2186
2187 if (info.type != OS_FILE_TYPE_FILE)
2188 continue;
2189
2190 const char *pname = strrchr(info.name, OS_PATH_SEPARATOR);
2191 if (!pname)
2192 pname = info.name;
2193
2194 if (!starts_with(pname, "aws-kms-key") &&
2195 !starts_with(pname, "aria_log"))
2196 /* For ES exchange the above line with the following code:
2197 (!xtrabackup_prepare || !xtrabackup_incremental_dir ||
2198 !starts_with(pname, "aria_log")))
2199 */
2200 continue;
2201
2202 if (xtrabackup_prepare && xtrabackup_incremental_dir &&
2203 file_exists(info.name))
2204 unlink(info.name);
2205
2206 std::string full_path(dir_path);
2207 full_path.append(1, OS_PATH_SEPARATOR).append(info.name);
2208 if (!(ret = copy_file(ds_data, full_path.c_str() , info.name, 1)))
2209 break;
2210 }
2211 os_file_closedir(dir);
2212 return ret;
2213 }
2214
2215
rocksdb_remove_checkpoint_directory()2216 static int rocksdb_remove_checkpoint_directory()
2217 {
2218 xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
2219 return 0;
2220 }
2221
has_rocksdb_plugin()2222 static bool has_rocksdb_plugin()
2223 {
2224 static bool first_time = true;
2225 static bool has_plugin= false;
2226 if (!first_time || !xb_backup_rocksdb)
2227 return has_plugin;
2228
2229 const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
2230 MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
2231 MYSQL_ROW row = mysql_fetch_row(result);
2232 if (row)
2233 has_plugin = !strcmp(row[0], "1");
2234 mysql_free_result(result);
2235 first_time = false;
2236 return has_plugin;
2237 }
2238
trim_trailing_dir_sep(char * path)2239 static char *trim_trailing_dir_sep(char *path)
2240 {
2241 size_t path_len = strlen(path);
2242 while (path_len)
2243 {
2244 char c = path[path_len - 1];
2245 if (c == '/' IF_WIN(|| c == '\\', ))
2246 path_len--;
2247 else
2248 break;
2249 }
2250 path[path_len] = 0;
2251 return path;
2252 }
2253
2254 /*
2255 Create a file hardlink.
2256 @return true on success, false on error.
2257 */
make_hardlink(const char * from_path,const char * to_path)2258 static bool make_hardlink(const char *from_path, const char *to_path)
2259 {
2260 DBUG_EXECUTE_IF("no_hardlinks", return false;);
2261 char to_path_full[FN_REFLEN];
2262 if (!is_abs_path(to_path))
2263 {
2264 fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
2265 }
2266 else
2267 {
2268 strncpy(to_path_full, to_path, sizeof(to_path_full));
2269 }
2270 #ifdef _WIN32
2271 return CreateHardLink(to_path_full, from_path, NULL);
2272 #else
2273 return !link(from_path, to_path_full);
2274 #endif
2275 }
2276
2277 /*
2278 Copies or moves a directory (non-recursively so far).
2279 Helper function used to backup rocksdb checkpoint, or copy-back the
2280 rocksdb files.
2281
2282 Has optimization that allows to use hardlinks when possible
2283 (source and destination are directories on the same device)
2284 */
copy_or_move_dir(const char * from,const char * to,bool do_copy,bool allow_hardlinks)2285 static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
2286 {
2287 datadir_node_t node;
2288 datadir_node_init(&node);
2289 datadir_iter_t *it = datadir_iter_new(from, false);
2290
2291 while (datadir_iter_next(it, &node))
2292 {
2293 char to_path[FN_REFLEN];
2294 const char *from_path = node.filepath;
2295 snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
2296 bool rc = false;
2297 if (do_copy && allow_hardlinks)
2298 {
2299 rc = make_hardlink(from_path, to_path);
2300 if (rc)
2301 {
2302 msg("Creating hardlink from %s to %s",from_path, to_path);
2303 }
2304 else
2305 {
2306 allow_hardlinks = false;
2307 }
2308 }
2309
2310 if (!rc)
2311 {
2312 rc = (do_copy ?
2313 copy_file(ds_data, from_path, to_path, 1) :
2314 move_file(ds_data, from_path, node.filepath_rel,
2315 to, 1));
2316 }
2317 if (!rc)
2318 die("copy or move file failed");
2319 }
2320 datadir_iter_free(it);
2321 datadir_node_free(&node);
2322
2323 }
2324
2325 /*
2326 Obtain user level lock , to protect the checkpoint directory of the server
2327 from being user/overwritten by different backup processes, if backups are
2328 running in parallel.
2329
2330 This lock will be acquired before rocksdb checkpoint is created, held
2331 while all files from it are being copied to their final backup destination,
2332 and finally released after the checkpoint is removed.
2333 */
rocksdb_lock_checkpoint()2334 static void rocksdb_lock_checkpoint()
2335 {
2336 msg("Obtaining rocksdb checkpoint lock.");
2337 MYSQL_RES *res =
2338 xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);
2339
2340 MYSQL_ROW r = mysql_fetch_row(res);
2341 if (r && r[0] && strcmp(r[0], "1"))
2342 {
2343 msg("Could not obtain rocksdb checkpont lock.");
2344 exit(EXIT_FAILURE);
2345 }
2346 mysql_free_result(res);
2347 }
2348
rocksdb_unlock_checkpoint()2349 static void rocksdb_unlock_checkpoint()
2350 {
2351 xb_mysql_query(mysql_connection,
2352 "SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
2353 }
2354
2355
2356 /*
2357 Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
2358 directory.
2359 A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
2360 acquired be this function.
2361 */
2362 #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
2363 static char rocksdb_checkpoint_dir[FN_REFLEN];
2364
rocksdb_create_checkpoint()2365 static void rocksdb_create_checkpoint()
2366 {
2367 MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
2368 MYSQL_ROW row = mysql_fetch_row(result);
2369
2370 DBUG_ASSERT(row && row[0] && row[1]);
2371
2372 char *rocksdbdir = row[0];
2373 char *datadir = row[1];
2374
2375 if (is_abs_path(rocksdbdir))
2376 {
2377 snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
2378 "%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
2379 }
2380 else
2381 {
2382 snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
2383 "%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
2384 trim_dotslash(rocksdbdir));
2385 }
2386 mysql_free_result(result);
2387
2388 #ifdef _WIN32
2389 for (char *p = rocksdb_checkpoint_dir; *p; p++)
2390 if (*p == '\\') *p = '/';
2391 #endif
2392
2393 rocksdb_lock_checkpoint();
2394
2395 if (!access(rocksdb_checkpoint_dir, 0))
2396 {
2397 msg("Removing rocksdb checkpoint from previous backup attempt.");
2398 rocksdb_remove_checkpoint_directory();
2399 }
2400
2401 char query[FN_REFLEN + 32];
2402 snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
2403 xb_mysql_query(mysql_connection, query, false, true);
2404 }
2405
2406 /*
2407 Copy files from rocksdb temporary checkpoint to final destination.
2408 remove temp.checkpoint directory (in server's datadir)
2409 and release user level lock acquired inside rocksdb_create_checkpoint().
2410 */
rocksdb_backup_checkpoint()2411 static void rocksdb_backup_checkpoint()
2412 {
2413 msg("Backing up rocksdb files.");
2414 char rocksdb_backup_dir[FN_REFLEN];
2415 snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
2416 bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
2417 if (backup_to_directory)
2418 {
2419 if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
2420 die("Can't create rocksdb backup directory %s", rocksdb_backup_dir);
2421 }
2422 }
2423 copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
2424 rocksdb_remove_checkpoint_directory();
2425 rocksdb_unlock_checkpoint();
2426 }
2427
2428 /*
2429 Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
2430 */
rocksdb_copy_back()2431 static void rocksdb_copy_back() {
2432 if (access(ROCKSDB_BACKUP_DIR, 0))
2433 return;
2434 char rocksdb_home_dir[FN_REFLEN];
2435 if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
2436 strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof rocksdb_home_dir - 1);
2437 rocksdb_home_dir[sizeof rocksdb_home_dir - 1] = '\0';
2438 } else {
2439 snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home,
2440 xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
2441 }
2442 mkdirp(rocksdb_home_dir, 0777, MYF(0));
2443 copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
2444 }
2445