1 /******************************************************
2 hot backup tool for InnoDB
3 (c) 2009-2015 Percona LLC and/or its affiliates
4 (c) 2017 MariaDB
5 Originally Created 3/3/2009 Yasufumi Kinoshita
6 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
7 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
8 
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; version 2 of the License.
12 
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 GNU General Public License for more details.
17 
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
21 
22 *******************************************************
23 
24 This file incorporates work covered by the following copyright and
25 permission notice:
26 
27 Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
28 
29 This program is free software; you can redistribute it and/or modify it under
30 the terms of the GNU General Public License as published by the Free Software
31 Foundation; version 2 of the License.
32 
33 This program is distributed in the hope that it will be useful, but WITHOUT
34 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
35 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
36 
37 You should have received a copy of the GNU General Public License along with
38 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
39 Street, Fifth Floor, Boston, MA 02110-1335 USA
40 
41 *******************************************************/
42 
43 #include <my_global.h>
44 #include <os0file.h>
45 #include <my_dir.h>
46 #include <ut0mem.h>
47 #include <srv0start.h>
48 #include <fil0fil.h>
49 #include <trx0sys.h>
50 #include <set>
51 #include <string>
52 #include <mysqld.h>
53 #include <sstream>
54 #include "fil_cur.h"
55 #include "xtrabackup.h"
56 #include "common.h"
57 #include "backup_copy.h"
58 #include "backup_mysql.h"
59 #include <btr0btr.h>
60 
61 #define ROCKSDB_BACKUP_DIR "#rocksdb"
62 
63 /* list of files to sync for --rsync mode */
64 static std::set<std::string> rsync_list;
65 /* locations of tablespaces read from .isl files */
66 static std::map<std::string, std::string> tablespace_locations;
67 
68 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
69 bool binlog_locked;
70 
71 static void rocksdb_create_checkpoint();
72 static bool has_rocksdb_plugin();
73 static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
74 static void rocksdb_backup_checkpoint();
75 static void rocksdb_copy_back();
76 
is_abs_path(const char * path)77 static bool is_abs_path(const char *path)
78 {
79 #ifdef _WIN32
80 	return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
81 #else
82 	return path[0] == '/';
83 #endif
84 }
85 
86 /************************************************************************
87 Struct represents file or directory. */
88 struct datadir_node_t {
89 	ulint		dbpath_len;
90 	char		*filepath;
91 	ulint		filepath_len;
92 	char		*filepath_rel;
93 	ulint		filepath_rel_len;
94 	bool		is_empty_dir;
95 	bool		is_file;
96 };
97 
98 /************************************************************************
99 Holds the state needed to enumerate files in MySQL data directory. */
100 struct datadir_iter_t {
101 	char		*datadir_path;
102 	char		*dbpath;
103 	ulint		dbpath_len;
104 	char		*filepath;
105 	ulint		filepath_len;
106 	char		*filepath_rel;
107 	ulint		filepath_rel_len;
108 	pthread_mutex_t	mutex;
109 	os_file_dir_t	dir;
110 	os_file_dir_t	dbdir;
111 	os_file_stat_t	dbinfo;
112 	os_file_stat_t	fileinfo;
113 	dberr_t		err;
114 	bool		is_empty_dir;
115 	bool		is_file;
116 	bool		skip_first_level;
117 };
118 
119 
120 /************************************************************************
121 Represents the context of the thread processing MySQL data directory. */
122 struct datadir_thread_ctxt_t {
123 	datadir_iter_t		*it;
124 	uint			n_thread;
125 	uint			*count;
126 	pthread_mutex_t*	count_mutex;
127 	os_thread_id_t		id;
128 	bool			ret;
129 };
130 
131 static bool backup_files_from_datadir(const char *dir_path);
132 
133 /************************************************************************
134 Retirn true if character if file separator */
135 bool
is_path_separator(char c)136 is_path_separator(char c)
137 {
138 	return(c == FN_LIBCHAR || c == FN_LIBCHAR2);
139 }
140 
141 
142 /************************************************************************
143 Fill the node struct. Memory for node need to be allocated and freed by
144 the caller. It is caller responsibility to initialize node with
145 datadir_node_init and cleanup the memory with datadir_node_free.
146 Node can not be shared between threads. */
147 static
148 void
datadir_node_fill(datadir_node_t * node,datadir_iter_t * it)149 datadir_node_fill(datadir_node_t *node, datadir_iter_t *it)
150 {
151 	if (node->filepath_len < it->filepath_len) {
152 		free(node->filepath);
153 		node->filepath = (char*)(malloc(it->filepath_len));
154 		node->filepath_len = it->filepath_len;
155 	}
156 	if (node->filepath_rel_len < it->filepath_rel_len) {
157 		free(node->filepath_rel);
158 		node->filepath_rel = (char*)(malloc(it->filepath_rel_len));
159 		node->filepath_rel_len = it->filepath_rel_len;
160 	}
161 
162 	strcpy(node->filepath, it->filepath);
163 	strcpy(node->filepath_rel, it->filepath_rel);
164 	node->is_empty_dir = it->is_empty_dir;
165 	node->is_file = it->is_file;
166 }
167 
168 static
169 void
datadir_node_free(datadir_node_t * node)170 datadir_node_free(datadir_node_t *node)
171 {
172 	free(node->filepath);
173 	free(node->filepath_rel);
174 	memset(node, 0, sizeof(datadir_node_t));
175 }
176 
177 static
178 void
datadir_node_init(datadir_node_t * node)179 datadir_node_init(datadir_node_t *node)
180 {
181 	memset(node, 0, sizeof(datadir_node_t));
182 }
183 
184 
185 /************************************************************************
186 Create the MySQL data directory iterator. Memory needs to be released
187 with datadir_iter_free. Position should be advanced with
188 datadir_iter_next_file. Iterator can be shared between multiple
189 threads. It is guaranteed that each thread receives unique file from
190 data directory into its local node struct. */
191 static
192 datadir_iter_t *
datadir_iter_new(const char * path,bool skip_first_level=true)193 datadir_iter_new(const char *path, bool skip_first_level = true)
194 {
195 	datadir_iter_t *it;
196 
197 	it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t)));
198 	if (!it)
199 		goto error;
200 	memset(it, 0, sizeof(datadir_iter_t));
201 
202 	pthread_mutex_init(&it->mutex, NULL);
203 	it->datadir_path = strdup(path);
204 
205 	it->dir = os_file_opendir(it->datadir_path);
206 
207 	if (it->dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
208 
209 		goto error;
210 	}
211 
212 	it->err = DB_SUCCESS;
213 
214 	it->dbpath_len = FN_REFLEN;
215 	it->dbpath = static_cast<char*>(malloc(it->dbpath_len));
216 
217 	it->filepath_len = FN_REFLEN;
218 	it->filepath = static_cast<char*>(malloc(it->filepath_len));
219 
220 	it->filepath_rel_len = FN_REFLEN;
221 	it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len));
222 
223 	it->skip_first_level = skip_first_level;
224 
225 	return(it);
226 
227 error:
228 	free(it);
229 
230 	return(NULL);
231 }
232 
233 static
234 bool
datadir_iter_next_database(datadir_iter_t * it)235 datadir_iter_next_database(datadir_iter_t *it)
236 {
237 	if (it->dbdir != NULL) {
238 		if (os_file_closedir_failed(it->dbdir)) {
239 			msg("Warning: could not"
240 			      " close database directory %s", it->dbpath);
241 			it->err = DB_ERROR;
242 
243 		}
244 		it->dbdir = NULL;
245 	}
246 
247 	while (os_file_readdir_next_file(it->datadir_path,
248 					  it->dir, &it->dbinfo) == 0) {
249 		ulint	len;
250 
251 		if ((it->dbinfo.type == OS_FILE_TYPE_FILE
252 		     && it->skip_first_level)
253 		    || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
254 
255 			continue;
256 		}
257 
258 		/* We found a symlink or a directory; try opening it to see
259 		if a symlink is a directory */
260 
261 		len = strlen(it->datadir_path)
262 			+ strlen (it->dbinfo.name) + 2;
263 		if (len > it->dbpath_len) {
264 			it->dbpath_len = len;
265 			free(it->dbpath);
266 
267 			it->dbpath = static_cast<char*>(
268 				malloc(it->dbpath_len));
269 		}
270 		snprintf(it->dbpath, it->dbpath_len, "%s/%s",
271 			 it->datadir_path, it->dbinfo.name);
272 		os_normalize_path(it->dbpath);
273 
274 		if (it->dbinfo.type == OS_FILE_TYPE_FILE) {
275 			it->is_file = true;
276 			return(true);
277 		}
278 
279 		if (check_if_skip_database_by_path(it->dbpath)) {
280 			msg("Skipping db: %s", it->dbpath);
281 			continue;
282 		}
283 
284 		/* We want wrong directory permissions to be a fatal error for
285 		XtraBackup. */
286 		it->dbdir = os_file_opendir(it->dbpath);
287 
288 		if (it->dir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
289 			it->is_file = false;
290 			return(true);
291 		}
292 
293 	}
294 
295 	return(false);
296 }
297 
298 /************************************************************************
299 Concatenate n parts into single path */
300 static
301 void
make_path_n(int n,char ** path,ulint * path_len,...)302 make_path_n(int n, char **path, ulint *path_len, ...)
303 {
304 	ulint len_needed = n + 1;
305 	char *p;
306 	int i;
307 	va_list vl;
308 
309 	ut_ad(n > 0);
310 
311 	va_start(vl, path_len);
312 	for (i = 0; i < n; i++) {
313 		p = va_arg(vl, char*);
314 		len_needed += strlen(p);
315 	}
316 	va_end(vl);
317 
318 	if (len_needed < *path_len) {
319 		free(*path);
320 		*path = static_cast<char*>(malloc(len_needed));
321 	}
322 
323 	va_start(vl, path_len);
324 	p = va_arg(vl, char*);
325 	strcpy(*path, p);
326 	for (i = 1; i < n; i++) {
327 		size_t plen;
328 		p = va_arg(vl, char*);
329 		plen = strlen(*path);
330 		if (!is_path_separator((*path)[plen - 1])) {
331 			(*path)[plen] = FN_LIBCHAR;
332 			(*path)[plen + 1] = 0;
333 		}
334 		strcat(*path + plen, p);
335 	}
336 	va_end(vl);
337 }
338 
339 static
340 bool
datadir_iter_next_file(datadir_iter_t * it)341 datadir_iter_next_file(datadir_iter_t *it)
342 {
343 	if (it->is_file && it->dbpath) {
344 		make_path_n(2, &it->filepath, &it->filepath_len,
345 				it->datadir_path, it->dbinfo.name);
346 
347 		make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
348 				it->dbinfo.name);
349 
350 		it->is_empty_dir = false;
351 		it->is_file = false;
352 
353 		return(true);
354 	}
355 
356 	if (!it->dbpath || !it->dbdir) {
357 
358 		return(false);
359 	}
360 
361 	while (os_file_readdir_next_file(it->dbpath, it->dbdir,
362 					  &it->fileinfo) == 0) {
363 
364 		if (it->fileinfo.type == OS_FILE_TYPE_DIR) {
365 
366 			continue;
367 		}
368 
369 		/* We found a symlink or a file */
370 		make_path_n(3, &it->filepath, &it->filepath_len,
371 				it->datadir_path, it->dbinfo.name,
372 				it->fileinfo.name);
373 
374 		make_path_n(2, &it->filepath_rel, &it->filepath_rel_len,
375 				it->dbinfo.name, it->fileinfo.name);
376 
377 		it->is_empty_dir = false;
378 
379 		return(true);
380 	}
381 
382 	return(false);
383 }
384 
385 static
386 bool
datadir_iter_next(datadir_iter_t * it,datadir_node_t * node)387 datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
388 {
389 	bool	ret = true;
390 
391 	pthread_mutex_lock(&it->mutex);
392 
393 	if (datadir_iter_next_file(it)) {
394 
395 		datadir_node_fill(node, it);
396 
397 		goto done;
398 	}
399 
400 	while (datadir_iter_next_database(it)) {
401 
402 		if (datadir_iter_next_file(it)) {
403 
404 			datadir_node_fill(node, it);
405 
406 			goto done;
407 		}
408 
409 		make_path_n(2, &it->filepath, &it->filepath_len,
410 				it->datadir_path, it->dbinfo.name);
411 
412 		make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
413 				it->dbinfo.name);
414 
415 		it->is_empty_dir = true;
416 
417 		datadir_node_fill(node, it);
418 
419 		goto done;
420 	}
421 
422 	/* nothing found */
423 	ret = false;
424 
425 done:
426 	pthread_mutex_unlock(&it->mutex);
427 
428 	return(ret);
429 }
430 
431 /************************************************************************
432 Interface to read MySQL data file sequentially. One should open file
433 with datafile_open to get cursor and close the cursor with
434 datafile_close. Cursor can not be shared between multiple
435 threads. */
436 static
437 void
datadir_iter_free(datadir_iter_t * it)438 datadir_iter_free(datadir_iter_t *it)
439 {
440 	pthread_mutex_destroy(&it->mutex);
441 
442 	if (it->dbdir) {
443 
444 		os_file_closedir(it->dbdir);
445 	}
446 
447 	if (it->dir) {
448 
449 		os_file_closedir(it->dir);
450 	}
451 
452 	free(it->dbpath);
453 	free(it->filepath);
454 	free(it->filepath_rel);
455 	free(it->datadir_path);
456 	free(it);
457 }
458 
459 
460 /************************************************************************
461 Holds the state needed to copy single data file. */
462 struct datafile_cur_t {
463 	pfs_os_file_t	file;
464 	char		rel_path[FN_REFLEN];
465 	char		abs_path[FN_REFLEN];
466 	MY_STAT		statinfo;
467 	uint		thread_n;
468 	byte*		orig_buf;
469 	byte*		buf;
470 	size_t		buf_size;
471 	size_t		buf_read;
472 	size_t		buf_offset;
473 
datafile_cur_tdatafile_cur_t474 	explicit datafile_cur_t(const char* filename = NULL) :
475 		file(), thread_n(0), orig_buf(NULL), buf(NULL), buf_size(0),
476 		buf_read(0), buf_offset(0)
477 	{
478 		memset(rel_path, 0, sizeof rel_path);
479 		if (filename) {
480 			strncpy(abs_path, filename, sizeof abs_path - 1);
481 			abs_path[(sizeof abs_path) - 1] = 0;
482 		} else {
483 			abs_path[0] = '\0';
484 		}
485 		rel_path[0] = '\0';
486 		memset(&statinfo, 0, sizeof statinfo);
487 	}
488 };
489 
490 static
491 void
datafile_close(datafile_cur_t * cursor)492 datafile_close(datafile_cur_t *cursor)
493 {
494 	if (cursor->file != OS_FILE_CLOSED) {
495 		os_file_close(cursor->file);
496 	}
497 	free(cursor->buf);
498 }
499 
500 static
501 bool
datafile_open(const char * file,datafile_cur_t * cursor,uint thread_n)502 datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
503 {
504 	bool		success;
505 
506 	new (cursor) datafile_cur_t(file);
507 
508 	/* Get the relative path for the destination tablespace name, i.e. the
509 	one that can be appended to the backup root directory. Non-system
510 	tablespaces may have absolute paths for remote tablespaces in MySQL
511 	5.6+. We want to make "local" copies for the backup. */
512 	strncpy(cursor->rel_path,
513 		xb_get_relative_path(cursor->abs_path, FALSE),
514 		(sizeof cursor->rel_path) - 1);
515 	cursor->rel_path[(sizeof cursor->rel_path) - 1] = '\0';
516 
517 	cursor->file = os_file_create_simple_no_error_handling(
518 		0, cursor->abs_path,
519 		OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success);
520 	if (!success) {
521 		/* The following call prints an error message */
522 		os_file_get_last_error(TRUE);
523 
524 		msg(thread_n,"error: cannot open "
525 			"file %s", cursor->abs_path);
526 
527 		return(false);
528 	}
529 
530 	if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) {
531 		msg(thread_n, "error: cannot stat %s", cursor->abs_path);
532 		datafile_close(cursor);
533 		return(false);
534 	}
535 
536 	posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
537 
538 	cursor->buf_size = 10 * 1024 * 1024;
539 	cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size));
540 
541 	return(true);
542 }
543 
544 
545 static
546 xb_fil_cur_result_t
datafile_read(datafile_cur_t * cursor)547 datafile_read(datafile_cur_t *cursor)
548 {
549 	ulint		to_read;
550 
551 	xtrabackup_io_throttling();
552 
553 	to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset,
554 		      cursor->buf_size);
555 
556 	if (to_read == 0) {
557 		return(XB_FIL_CUR_EOF);
558 	}
559 
560 	if (os_file_read(IORequestRead,
561 			  cursor->file, cursor->buf, cursor->buf_offset,
562 			  to_read) != DB_SUCCESS) {
563 		return(XB_FIL_CUR_ERROR);
564 	}
565 
566 	posix_fadvise(cursor->file, cursor->buf_offset, to_read,
567 		      POSIX_FADV_DONTNEED);
568 
569 	cursor->buf_read = to_read;
570 	cursor->buf_offset += to_read;
571 
572 	return(XB_FIL_CUR_SUCCESS);
573 }
574 
575 
576 
577 /************************************************************************
578 Check to see if a file exists.
579 Takes name of the file to check.
580 @return true if file exists. */
581 static
582 bool
file_exists(const char * filename)583 file_exists(const char *filename)
584 {
585 	MY_STAT stat_arg;
586 
587 	if (!my_stat(filename, &stat_arg, MYF(0))) {
588 
589 		return(false);
590 	}
591 
592 	return(true);
593 }
594 
595 /************************************************************************
596 Trim leading slashes from absolute path so it becomes relative */
597 static
598 const char *
trim_dotslash(const char * path)599 trim_dotslash(const char *path)
600 {
601 	while (*path) {
602 		if (is_path_separator(*path)) {
603 			++path;
604 			continue;
605 		}
606 		if (*path == '.' && is_path_separator(path[1])) {
607 			path += 2;
608 			continue;
609 		}
610 		break;
611 	}
612 
613 	return(path);
614 }
615 
616 
617 
618 /************************************************************************
619 Check if string ends with given suffix.
620 @return true if string ends with given suffix. */
621 bool
ends_with(const char * str,const char * suffix)622 ends_with(const char *str, const char *suffix)
623 {
624 	size_t suffix_len = strlen(suffix);
625 	size_t str_len = strlen(str);
626 	return(str_len >= suffix_len
627 	       && strcmp(str + str_len - suffix_len, suffix) == 0);
628 }
629 
starts_with(const char * str,const char * prefix)630 static bool starts_with(const char *str, const char *prefix)
631 {
632 	return strncmp(str, prefix, strlen(prefix)) == 0;
633 }
634 
635 /************************************************************************
636 Create directories recursively.
637 @return 0 if directories created successfully. */
638 static
639 int
mkdirp(const char * pathname,int Flags,myf MyFlags)640 mkdirp(const char *pathname, int Flags, myf MyFlags)
641 {
642 	char *parent, *p;
643 
644 	/* make a parent directory path */
645 	if (!(parent= strdup(pathname)))
646           return(-1);
647 
648 	for (p = parent + strlen(parent);
649 	    !is_path_separator(*p) && p != parent; p--) ;
650 
651 	*p = 0;
652 
653 	/* try to make parent directory */
654 	if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) {
655 		free(parent);
656 		return(-1);
657 	}
658 
659 	/* make this one if parent has been made */
660 	if (my_mkdir(pathname, Flags, MyFlags) == 0) {
661 		free(parent);
662 		return(0);
663 	}
664 
665 	/* if it already exists that is fine */
666 	if (errno == EEXIST) {
667 		free(parent);
668 		return(0);
669 	}
670 
671 	free(parent);
672 	return(-1);
673 }
674 
675 /************************************************************************
676 Return true if first and second arguments are the same path. */
677 bool
equal_paths(const char * first,const char * second)678 equal_paths(const char *first, const char *second)
679 {
680 #ifdef HAVE_REALPATH
681 	char *real_first, *real_second;
682 	int result;
683 
684 	real_first = realpath(first, 0);
685 	if (real_first == NULL) {
686 		return false;
687 	}
688 
689 	real_second = realpath(second, 0);
690 	if (real_second == NULL) {
691 		free(real_first);
692 		return false;
693 	}
694 
695 	result = strcmp(real_first, real_second);
696 	free(real_first);
697 	free(real_second);
698 	return result == 0;
699 #else
700 	return strcmp(first, second) == 0;
701 #endif
702 }
703 
704 /************************************************************************
705 Check if directory exists. Optionally create directory if doesn't
706 exist.
707 @return true if directory exists and if it was created successfully. */
708 bool
directory_exists(const char * dir,bool create)709 directory_exists(const char *dir, bool create)
710 {
711 	os_file_dir_t os_dir;
712 	MY_STAT stat_arg;
713 	char errbuf[MYSYS_STRERROR_SIZE];
714 
715 	if (my_stat(dir, &stat_arg, MYF(0)) == NULL) {
716 
717 		if (!create) {
718 			return(false);
719 		}
720 
721 		if (mkdirp(dir, 0777, MYF(0)) < 0) {
722 			my_strerror(errbuf, sizeof(errbuf), my_errno);
723 			msg("Can not create directory %s: %s", dir, errbuf);
724 			return(false);
725 		}
726 	}
727 
728 	/* could be symlink */
729 	os_dir = os_file_opendir(dir);
730 
731 	if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
732 		my_strerror(errbuf, sizeof(errbuf), my_errno);
733 		msg("Can not open directory %s: %s", dir,
734 			errbuf);
735 
736 		return(false);
737 	}
738 
739 	os_file_closedir(os_dir);
740 
741 	return(true);
742 }
743 
744 /************************************************************************
745 Check that directory exists and it is empty. */
746 static
747 bool
directory_exists_and_empty(const char * dir,const char * comment)748 directory_exists_and_empty(const char *dir, const char *comment)
749 {
750 	os_file_dir_t os_dir;
751 	dberr_t err;
752 	os_file_stat_t info;
753 	bool empty;
754 
755 	if (!directory_exists(dir, true)) {
756 		return(false);
757 	}
758 
759 	os_dir = os_file_opendir(dir);
760 
761 	if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
762 		msg("%s can not open directory %s", comment, dir);
763 		return(false);
764 	}
765 
766 	empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0);
767 
768 	os_file_closedir(os_dir);
769 
770 	if (!empty) {
771 		msg("%s directory %s is not empty!", comment, dir);
772 	}
773 
774 	return(empty);
775 }
776 
777 
778 /************************************************************************
779 Check if file name ends with given set of suffixes.
780 @return true if it does. */
781 static
782 bool
filename_matches(const char * filename,const char ** ext_list)783 filename_matches(const char *filename, const char **ext_list)
784 {
785 	const char **ext;
786 
787 	for (ext = ext_list; *ext; ext++) {
788 		if (ends_with(filename, *ext)) {
789 			return(true);
790 		}
791 	}
792 
793 	return(false);
794 }
795 
796 
797 /************************************************************************
798 Copy data file for backup. Also check if it is allowed to copy by
799 comparing its name to the list of known data file types and checking
800 if passes the rules for partial backup.
801 @return true if file backed up or skipped successfully. */
802 static
803 bool
datafile_copy_backup(const char * filepath,uint thread_n)804 datafile_copy_backup(const char *filepath, uint thread_n)
805 {
806 	const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
807 		"MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
808 		NULL};
809 
810 	/* Get the name and the path for the tablespace. node->name always
811 	contains the path (which may be absolute for remote tablespaces in
812 	5.6+). space->name contains the tablespace name in the form
813 	"./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
814 	multi-node shared tablespace, space->name contains the name of the first
815 	node, but that's irrelevant, since we only need node_name to match them
816 	against filters, and the shared tablespace is always copied regardless
817 	of the filters value. */
818 
819 	if (check_if_skip_table(filepath)) {
820 		msg(thread_n,"Skipping %s.", filepath);
821 		return(true);
822 	}
823 
824 	if (filename_matches(filepath, ext_list)) {
825 		return copy_file(ds_data, filepath, filepath, thread_n);
826 	}
827 
828 	return(true);
829 }
830 
831 
832 /************************************************************************
833 Same as datafile_copy_backup, but put file name into the list for
834 rsync command. */
835 static
836 bool
datafile_rsync_backup(const char * filepath,bool save_to_list,FILE * f)837 datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
838 {
839 	const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
840 		"MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
841 		NULL};
842 
843 	/* Get the name and the path for the tablespace. node->name always
844 	contains the path (which may be absolute for remote tablespaces in
845 	5.6+). space->name contains the tablespace name in the form
846 	"./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
847 	multi-node shared tablespace, space->name contains the name of the first
848 	node, but that's irrelevant, since we only need node_name to match them
849 	against filters, and the shared tablespace is always copied regardless
850 	of the filters value. */
851 
852 	if (check_if_skip_table(filepath)) {
853 		return(true);
854 	}
855 
856 	if (filename_matches(filepath, ext_list)) {
857 		fprintf(f, "%s\n", filepath);
858 		if (save_to_list) {
859 			rsync_list.insert(filepath);
860 		}
861 	}
862 
863 	return(true);
864 }
865 
backup_file_print_buf(const char * filename,const char * buf,int buf_len)866 bool backup_file_print_buf(const char *filename, const char *buf, int buf_len)
867 {
868 	ds_file_t	*dstfile	= NULL;
869 	MY_STAT		 stat;			/* unused for now */
870 	const char	*action;
871 
872 	memset(&stat, 0, sizeof(stat));
873 
874 	stat.st_size = buf_len;
875 	stat.st_mtime = my_time(0);
876 
877 	dstfile = ds_open(ds_data, filename, &stat);
878 	if (dstfile == NULL) {
879 		msg("error: Can't open the destination stream for %s",
880 			filename);
881 		goto error;
882 	}
883 
884 	action = xb_get_copy_action("Writing");
885 	msg("%s %s", action, filename);
886 
887 	if (buf_len == -1) {
888 		goto error;
889 	}
890 
891 	if (ds_write(dstfile, buf, buf_len)) {
892 		goto error;
893 	}
894 
895 	/* close */
896 	msg("        ...done");
897 
898 	if (ds_close(dstfile)) {
899 		goto error_close;
900 	}
901 
902 	return(true);
903 
904 error:
905 	if (dstfile != NULL) {
906 		ds_close(dstfile);
907 	}
908 
909 error_close:
910 	msg("Error: backup file failed.");
911 	return(false); /*ERROR*/
912 
913 	return true;
914 };
915 
916 static
917 bool
backup_file_vprintf(const char * filename,const char * fmt,va_list ap)918 backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
919 {
920 	char		*buf		= 0;
921 	int		 buf_len;
922 	buf_len = vasprintf(&buf, fmt, ap);
923 	bool result = backup_file_print_buf(filename, buf, buf_len);
924 	free(buf);
925 	return result;
926 }
927 
928 bool
backup_file_printf(const char * filename,const char * fmt,...)929 backup_file_printf(const char *filename, const char *fmt, ...)
930 {
931 	bool result;
932 	va_list ap;
933 
934 	va_start(ap, fmt);
935 
936 	result = backup_file_vprintf(filename, fmt, ap);
937 
938 	va_end(ap);
939 
940 	return(result);
941 }
942 
943 static
944 bool
run_data_threads(datadir_iter_t * it,os_thread_func_t func,uint n)945 run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
946 {
947 	datadir_thread_ctxt_t	*data_threads;
948 	uint			i, count;
949 	pthread_mutex_t		count_mutex;
950 	bool			ret;
951 
952 	data_threads = (datadir_thread_ctxt_t*)
953 		malloc(sizeof(datadir_thread_ctxt_t) * n);
954 
955 	pthread_mutex_init(&count_mutex, NULL);
956 	count = n;
957 
958 	for (i = 0; i < n; i++) {
959 		data_threads[i].it = it;
960 		data_threads[i].n_thread = i + 1;
961 		data_threads[i].count = &count;
962 		data_threads[i].count_mutex = &count_mutex;
963 		os_thread_create(func, data_threads + i, &data_threads[i].id);
964 	}
965 
966 	/* Wait for threads to exit */
967 	while (1) {
968 		os_thread_sleep(100000);
969 		pthread_mutex_lock(&count_mutex);
970 		if (count == 0) {
971 			pthread_mutex_unlock(&count_mutex);
972 			break;
973 		}
974 		pthread_mutex_unlock(&count_mutex);
975 	}
976 
977 	pthread_mutex_destroy(&count_mutex);
978 
979 	ret = true;
980 	for (i = 0; i < n; i++) {
981 		ret = data_threads[i].ret && ret;
982 		if (!data_threads[i].ret) {
983 			msg("Error: thread %u failed.", i);
984 		}
985 	}
986 
987 	free(data_threads);
988 
989 	return(ret);
990 }
991 
992 #ifdef _WIN32
993 #include <windows.h>
994 #include <accctrl.h>
995 #include <aclapi.h>
996 /*
997   On Windows, fix permission of the file after "copyback"
998   We assume that after copyback, mysqld will run as service as NetworkService
999   user, thus well give full permission on given file to that user.
1000 */
1001 
fix_win_file_permissions(const char * file)1002 static int fix_win_file_permissions(const char *file)
1003 {
1004 	struct {
1005 		TOKEN_USER tokenUser;
1006 		BYTE buffer[SECURITY_MAX_SID_SIZE];
1007 	} tokenInfoBuffer;
1008 	HANDLE hFile = CreateFile(file, READ_CONTROL | WRITE_DAC, 0, NULL, OPEN_EXISTING,
1009 		FILE_FLAG_BACKUP_SEMANTICS, NULL);
1010 	if (hFile == INVALID_HANDLE_VALUE)
1011 		return -1;
1012 	ACL* pOldDACL;
1013 	SECURITY_DESCRIPTOR* pSD = NULL;
1014 	EXPLICIT_ACCESS ea = { 0 };
1015 	PSID pSid = NULL;
1016 
1017 	GetSecurityInfo(hFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL,
1018 		&pOldDACL, NULL, (void**)&pSD);
1019 	DWORD size = SECURITY_MAX_SID_SIZE;
1020 	pSid = (PSID)tokenInfoBuffer.buffer;
1021 	if (!CreateWellKnownSid(WinNetworkServiceSid, NULL, pSid,
1022 		&size))
1023 	{
1024 		return 1;
1025 	}
1026 	ea.Trustee.TrusteeForm = TRUSTEE_IS_SID;
1027 	ea.Trustee.ptstrName = (LPTSTR)pSid;
1028 
1029 	ea.grfAccessMode = GRANT_ACCESS;
1030 	ea.grfAccessPermissions = GENERIC_ALL;
1031 	ea.grfInheritance = CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE;
1032 	ea.Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
1033 	ACL* pNewDACL = 0;
1034 	SetEntriesInAcl(1, &ea, pOldDACL, &pNewDACL);
1035 	if (pNewDACL)
1036 	{
1037 		SetSecurityInfo(hFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL,
1038 			pNewDACL, NULL);
1039 	}
1040 	if (pSD != NULL)
1041 		LocalFree((HLOCAL)pSD);
1042 	if (pNewDACL != NULL)
1043 		LocalFree((HLOCAL)pNewDACL);
1044 	CloseHandle(hFile);
1045 	return 0;
1046 }
1047 
1048 #endif
1049 
1050 
1051 /************************************************************************
1052 Copy file for backup/restore.
1053 @return true in case of success. */
1054 bool
copy_file(ds_ctxt_t * datasink,const char * src_file_path,const char * dst_file_path,uint thread_n)1055 copy_file(ds_ctxt_t *datasink,
1056 	  const char *src_file_path,
1057 	  const char *dst_file_path,
1058 	  uint thread_n)
1059 {
1060 	char			 dst_name[FN_REFLEN];
1061 	ds_file_t		*dstfile = NULL;
1062 	datafile_cur_t		 cursor;
1063 	xb_fil_cur_result_t	 res;
1064 	DBUG_ASSERT(datasink->datasink->remove);
1065 	const char	*dst_path =
1066 		(xtrabackup_copy_back || xtrabackup_move_back)?
1067 		dst_file_path : trim_dotslash(dst_file_path);
1068 
1069 	if (!datafile_open(src_file_path, &cursor, thread_n)) {
1070 		goto error_close;
1071 	}
1072 
1073 	strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
1074 
1075 	dstfile = ds_open(datasink, dst_path, &cursor.statinfo);
1076 	if (dstfile == NULL) {
1077 		msg(thread_n,"error: "
1078 			"cannot open the destination stream for %s", dst_name);
1079 		goto error;
1080 	}
1081 
1082 	msg(thread_n, "%s %s to %s", xb_get_copy_action(), src_file_path, dstfile->path);
1083 
1084 	/* The main copy loop */
1085 	while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
1086 
1087 		if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
1088 			goto error;
1089 		}
1090 		DBUG_EXECUTE_IF("copy_file_error", errno=ENOSPC;goto error;);
1091 	}
1092 
1093 	if (res == XB_FIL_CUR_ERROR) {
1094 		goto error;
1095 	}
1096 
1097 	/* close */
1098 	msg(thread_n,"        ...done");
1099 	datafile_close(&cursor);
1100 #ifdef _WIN32
1101 	if (xtrabackup_copy_back || xtrabackup_move_back)
1102 		ut_a(!fix_win_file_permissions(dstfile->path));
1103 #endif
1104 	if (ds_close(dstfile)) {
1105 		goto error_close;
1106 	}
1107 	return(true);
1108 
1109 error:
1110 	datafile_close(&cursor);
1111 	if (dstfile != NULL) {
1112 		datasink->datasink->remove(dstfile->path);
1113 		ds_close(dstfile);
1114 	}
1115 
1116 error_close:
1117 	msg(thread_n,"Error: copy_file() failed.");
1118 	return(false); /*ERROR*/
1119 }
1120 
1121 
1122 /************************************************************************
1123 Try to move file by renaming it. If source and destination are on
1124 different devices fall back to copy and unlink.
1125 @return true in case of success. */
1126 static
1127 bool
move_file(ds_ctxt_t * datasink,const char * src_file_path,const char * dst_file_path,const char * dst_dir,uint thread_n)1128 move_file(ds_ctxt_t *datasink,
1129 	  const char *src_file_path,
1130 	  const char *dst_file_path,
1131 	  const char *dst_dir, uint thread_n)
1132 {
1133 	char errbuf[MYSYS_STRERROR_SIZE];
1134 	char dst_file_path_abs[FN_REFLEN];
1135 	char dst_dir_abs[FN_REFLEN];
1136 	size_t dirname_length;
1137 
1138 	snprintf(dst_file_path_abs, sizeof(dst_file_path_abs),
1139 		 "%s/%s", dst_dir, dst_file_path);
1140 
1141 	dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length);
1142 
1143 	if (!directory_exists(dst_dir_abs, true)) {
1144 		return(false);
1145 	}
1146 
1147 	if (file_exists(dst_file_path_abs)) {
1148 		msg("Error: Move file %s to %s failed: Destination "
1149 			"file exists", src_file_path, dst_file_path_abs);
1150 		return(false);
1151 	}
1152 
1153 	msg(thread_n,"Moving %s to %s", src_file_path, dst_file_path_abs);
1154 
1155 	if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
1156 		if (my_errno == EXDEV) {
1157 			/* Fallback to copy/unlink */
1158 			if(!copy_file(datasink, src_file_path,
1159 					dst_file_path, thread_n))
1160 					return false;
1161 			msg(thread_n,"Removing %s", src_file_path);
1162 			if (unlink(src_file_path) != 0) {
1163 				my_strerror(errbuf, sizeof(errbuf), errno);
1164 				msg("Warning: unlink %s failed: %s",
1165 					src_file_path,
1166 					errbuf);
1167 			}
1168 			return true;
1169 		}
1170 		my_strerror(errbuf, sizeof(errbuf), my_errno);
1171 		msg("Can not move file %s to %s: %s",
1172 			src_file_path, dst_file_path_abs,
1173 			errbuf);
1174 		return(false);
1175 	}
1176 #ifdef _WIN32
1177 	if (xtrabackup_copy_back || xtrabackup_move_back)
1178 		ut_a(!fix_win_file_permissions(dst_file_path_abs));
1179 #endif
1180 	msg(thread_n,"        ...done");
1181 
1182 	return(true);
1183 }
1184 
1185 
1186 /************************************************************************
1187 Read link from .isl file if any and store it in the global map associated
1188 with given tablespace. */
1189 static
1190 void
read_link_file(const char * ibd_filepath,const char * link_filepath)1191 read_link_file(const char *ibd_filepath, const char *link_filepath)
1192 {
1193 	char *filepath= NULL;
1194 
1195 	FILE *file = fopen(link_filepath, "r+b");
1196 	if (file) {
1197 		filepath = static_cast<char*>(malloc(OS_FILE_MAX_PATH));
1198 
1199 		os_file_read_string(file, filepath, OS_FILE_MAX_PATH);
1200 		fclose(file);
1201 
1202 		if (strlen(filepath)) {
1203 			/* Trim whitespace from end of filepath */
1204 			ulint lastch = strlen(filepath) - 1;
1205 			while (lastch > 4 && filepath[lastch] <= 0x20) {
1206 				filepath[lastch--] = 0x00;
1207 			}
1208 			os_normalize_path(filepath);
1209 		}
1210 
1211 		tablespace_locations[ibd_filepath] = filepath;
1212 	}
1213 	free(filepath);
1214 }
1215 
1216 
1217 /************************************************************************
1218 Return the location of given .ibd if it was previously read
1219 from .isl file.
1220 @return NULL or destination .ibd file path. */
1221 static
1222 const char *
tablespace_filepath(const char * ibd_filepath)1223 tablespace_filepath(const char *ibd_filepath)
1224 {
1225 	std::map<std::string, std::string>::iterator it;
1226 
1227 	it = tablespace_locations.find(ibd_filepath);
1228 
1229 	if (it != tablespace_locations.end()) {
1230 		return it->second.c_str();
1231 	}
1232 
1233 	return NULL;
1234 }
1235 
1236 
1237 /************************************************************************
1238 Copy or move file depending on current mode.
1239 @return true in case of success. */
1240 static
1241 bool
copy_or_move_file(const char * src_file_path,const char * dst_file_path,const char * dst_dir,uint thread_n,bool copy=xtrabackup_copy_back)1242 copy_or_move_file(const char *src_file_path,
1243 		  const char *dst_file_path,
1244 		  const char *dst_dir,
1245 		  uint thread_n,
1246 		 bool copy = xtrabackup_copy_back)
1247 {
1248 	ds_ctxt_t *datasink = ds_data;		/* copy to datadir by default */
1249 	char filedir[FN_REFLEN];
1250 	size_t filedir_len;
1251 	bool ret;
1252 
1253 	/* read the link from .isl file */
1254 	if (ends_with(src_file_path, ".isl")) {
1255 		char *ibd_filepath;
1256 
1257 		ibd_filepath = strdup(src_file_path);
1258 		strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd");
1259 
1260 		read_link_file(ibd_filepath, src_file_path);
1261 
1262 		free(ibd_filepath);
1263 	}
1264 
1265 	/* check if there is .isl file */
1266 	if (ends_with(src_file_path, ".ibd")) {
1267 		char *link_filepath;
1268 		const char *filepath;
1269 
1270 		link_filepath = strdup(src_file_path);
1271 		strcpy(link_filepath + strlen(link_filepath) - 3, "isl");
1272 
1273 		read_link_file(src_file_path, link_filepath);
1274 
1275 		filepath = tablespace_filepath(src_file_path);
1276 
1277 		if (filepath != NULL) {
1278 			dirname_part(filedir, filepath, &filedir_len);
1279 
1280 			dst_file_path = filepath + filedir_len;
1281 			dst_dir = filedir;
1282 
1283 			if (!directory_exists(dst_dir, true)) {
1284 				ret = false;
1285 				free(link_filepath);
1286 				goto cleanup;
1287 			}
1288 
1289 			datasink = ds_create(dst_dir, DS_TYPE_LOCAL);
1290 		}
1291 
1292 		free(link_filepath);
1293 	}
1294 
1295 	ret = (copy ?
1296 		copy_file(datasink, src_file_path, dst_file_path, thread_n) :
1297 		move_file(datasink, src_file_path, dst_file_path,
1298 			  dst_dir, thread_n));
1299 
1300 cleanup:
1301 
1302 	if (datasink != ds_data) {
1303 		ds_destroy(datasink);
1304 	}
1305 
1306 	return(ret);
1307 }
1308 
1309 
1310 
1311 
1312 static
1313 bool
backup_files(const char * from,bool prep_mode)1314 backup_files(const char *from, bool prep_mode)
1315 {
1316 	char rsync_tmpfile_name[FN_REFLEN];
1317 	FILE *rsync_tmpfile = NULL;
1318 	datadir_iter_t *it;
1319 	datadir_node_t node;
1320 	bool ret = true;
1321 
1322 	if (prep_mode && !opt_rsync) {
1323 		return(true);
1324 	}
1325 
1326 	if (opt_rsync) {
1327 		snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1328 			"%s/%s%d", opt_mysql_tmpdir,
1329 			"xtrabackup_rsyncfiles_pass",
1330 			prep_mode ? 1 : 2);
1331 		rsync_tmpfile = fopen(rsync_tmpfile_name, "w");
1332 		if (rsync_tmpfile == NULL) {
1333 			msg("Error: can't create file %s",
1334 				rsync_tmpfile_name);
1335 			return(false);
1336 		}
1337 	}
1338 
1339 	msg("Starting %s non-InnoDB tables and files",
1340 	       prep_mode ? "prep copy of" : "to backup");
1341 
1342 	datadir_node_init(&node);
1343 	it = datadir_iter_new(from);
1344 
1345 	while (datadir_iter_next(it, &node)) {
1346 
1347 		if (!node.is_empty_dir) {
1348 			if (opt_rsync) {
1349 				ret = datafile_rsync_backup(node.filepath,
1350 					!prep_mode, rsync_tmpfile);
1351 			} else {
1352 				ret = datafile_copy_backup(node.filepath, 1);
1353 			}
1354 			if (!ret) {
1355 				msg("Failed to copy file %s", node.filepath);
1356 				goto out;
1357 			}
1358 		} else if (!prep_mode) {
1359 			/* backup fake file into empty directory */
1360 			char path[FN_REFLEN];
1361 			snprintf(path, sizeof(path),
1362 				 "%s/db.opt", node.filepath);
1363 			if (!(ret = backup_file_printf(
1364 					trim_dotslash(path), "%s", ""))) {
1365 				msg("Failed to create file %s", path);
1366 				goto out;
1367 			}
1368 		}
1369 	}
1370 
1371 	if (opt_rsync) {
1372 		std::stringstream cmd;
1373 		int err;
1374 
1375 		if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1376 			fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename);
1377 			rsync_list.insert(buffer_pool_filename);
1378 		}
1379 		if (file_exists("ib_lru_dump")) {
1380 			fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump");
1381 			rsync_list.insert("ib_lru_dump");
1382 		}
1383 
1384 		fclose(rsync_tmpfile);
1385 		rsync_tmpfile = NULL;
1386 
1387 		cmd << "rsync -t . --files-from=" << rsync_tmpfile_name
1388 		    << " " << xtrabackup_target_dir;
1389 
1390 		msg("Starting rsync as: %s", cmd.str().c_str());
1391 		if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) {
1392 			msg("Error: rsync failed with error code %d", err);
1393 			ret = false;
1394 			goto out;
1395 		}
1396 		msg("rsync finished successfully.");
1397 
1398 		if (!prep_mode && !opt_no_lock) {
1399 			char path[FN_REFLEN];
1400 			char dst_path[FN_REFLEN];
1401 			char *newline;
1402 
1403 			/* Remove files that have been removed between first and
1404 			second passes. Cannot use "rsync --delete" because it
1405 			does not work with --files-from. */
1406 			snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1407 				"%s/%s", opt_mysql_tmpdir,
1408 				"xtrabackup_rsyncfiles_pass1");
1409 
1410 			rsync_tmpfile = fopen(rsync_tmpfile_name, "r");
1411 			if (rsync_tmpfile == NULL) {
1412 				msg("Error: can't open file %s",
1413 					rsync_tmpfile_name);
1414 				ret = false;
1415 				goto out;
1416 			}
1417 
1418 			while (fgets(path, sizeof(path), rsync_tmpfile)) {
1419 
1420 				newline = strchr(path, '\n');
1421 				if (newline) {
1422 					*newline = 0;
1423 				}
1424 				if (rsync_list.count(path) < 1) {
1425 					snprintf(dst_path, sizeof(dst_path),
1426 						"%s/%s", xtrabackup_target_dir,
1427 						path);
1428 					msg("Removing %s", dst_path);
1429 					unlink(dst_path);
1430 				}
1431 			}
1432 
1433 			fclose(rsync_tmpfile);
1434 			rsync_tmpfile = NULL;
1435 		}
1436 	}
1437 
1438 	msg("Finished %s non-InnoDB tables and files",
1439 	       prep_mode ? "a prep copy of" : "backing up");
1440 
1441 out:
1442 	datadir_iter_free(it);
1443 	datadir_node_free(&node);
1444 
1445 	if (rsync_tmpfile != NULL) {
1446 		fclose(rsync_tmpfile);
1447 	}
1448 
1449 	return(ret);
1450 }
1451 
1452 void backup_fix_ddl(CorruptedPages &);
1453 
get_current_lsn(MYSQL * connection)1454 lsn_t get_current_lsn(MYSQL *connection)
1455 {
1456 	static const char lsn_prefix[] = "\nLog sequence number ";
1457 	lsn_t lsn = 0;
1458 	if (MYSQL_RES *res = xb_mysql_query(connection,
1459 					    "SHOW ENGINE INNODB STATUS",
1460 					    true, false)) {
1461 		if (MYSQL_ROW row = mysql_fetch_row(res)) {
1462 			const char *p= strstr(row[2], lsn_prefix);
1463 			DBUG_ASSERT(p);
1464 			if (p) {
1465 				p += sizeof lsn_prefix - 1;
1466 				lsn = lsn_t(strtoll(p, NULL, 10));
1467 			}
1468 		}
1469 		mysql_free_result(res);
1470 	}
1471 	return lsn;
1472 }
1473 
1474 lsn_t server_lsn_after_lock;
1475 extern void backup_wait_for_lsn(lsn_t lsn);
1476 /** Start --backup */
backup_start(CorruptedPages & corrupted_pages)1477 bool backup_start(CorruptedPages &corrupted_pages)
1478 {
1479 	if (!opt_no_lock) {
1480 		if (opt_safe_slave_backup) {
1481 			if (!wait_for_safe_slave(mysql_connection)) {
1482 				return(false);
1483 			}
1484 		}
1485 
1486 		if (!backup_files(fil_path_to_mysql_datadir, true)) {
1487 			return(false);
1488 		}
1489 
1490 		history_lock_time = time(NULL);
1491 
1492 		if (!lock_tables(mysql_connection)) {
1493 			return(false);
1494 		}
1495 		server_lsn_after_lock = get_current_lsn(mysql_connection);
1496 	}
1497 
1498 	if (!backup_files(fil_path_to_mysql_datadir, false)) {
1499 		return(false);
1500 	}
1501 
1502 	if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) {
1503 		return false;
1504 	}
1505 
1506 	if (has_rocksdb_plugin()) {
1507 		rocksdb_create_checkpoint();
1508 	}
1509 
1510 	msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock);
1511 	backup_wait_for_lsn(server_lsn_after_lock);
1512 	backup_fix_ddl(corrupted_pages);
1513 
1514 	// There is no need to stop slave thread before coping non-Innodb data when
1515 	// --no-lock option is used because --no-lock option requires that no DDL or
1516 	// DML to non-transaction tables can occur.
1517 	if (opt_no_lock) {
1518 		if (opt_safe_slave_backup) {
1519 			if (!wait_for_safe_slave(mysql_connection)) {
1520 				return(false);
1521 			}
1522 		}
1523 	}
1524 
1525 	if (opt_slave_info) {
1526 		lock_binlog_maybe(mysql_connection);
1527 
1528 		if (!write_slave_info(mysql_connection)) {
1529 			return(false);
1530 		}
1531 	}
1532 
1533 	/* The only reason why Galera/binlog info is written before
1534 	wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup
1535 	binary will start streamig a temporary copy of REDO log to stdout and
1536 	thus, any streaming from innobackupex would interfere. The only way to
1537 	avoid that is to have a single process, i.e. merge innobackupex and
1538 	xtrabackup. */
1539 	if (opt_galera_info) {
1540 		if (!write_galera_info(mysql_connection)) {
1541 			return(false);
1542 		}
1543 		write_current_binlog_file(mysql_connection);
1544 	}
1545 
1546 	if (opt_binlog_info == BINLOG_INFO_ON) {
1547 
1548 		lock_binlog_maybe(mysql_connection);
1549 		write_binlog_info(mysql_connection);
1550 	}
1551 
1552 	if (have_flush_engine_logs && !opt_no_lock) {
1553 		msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...");
1554 		xb_mysql_query(mysql_connection,
1555 			"FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false);
1556 	}
1557 
1558 	return(true);
1559 }
1560 
1561 /** Release resources after backup_start() */
backup_release()1562 void backup_release()
1563 {
1564 	/* release all locks */
1565 	if (!opt_no_lock) {
1566 		unlock_all(mysql_connection);
1567 		history_lock_time = 0;
1568 	} else {
1569 		history_lock_time = time(NULL) - history_lock_time;
1570 	}
1571 
1572 	if (opt_lock_ddl_per_table) {
1573 		mdl_unlock_all();
1574 	}
1575 
1576 	if (opt_safe_slave_backup && sql_thread_started) {
1577 		msg("Starting slave SQL thread");
1578 		xb_mysql_query(mysql_connection,
1579 				"START SLAVE SQL_THREAD", false);
1580 	}
1581 }
1582 
1583 /** Finish after backup_start() and backup_release() */
backup_finish()1584 bool backup_finish()
1585 {
1586 	/* Copy buffer pool dump or LRU dump */
1587 	if (!opt_rsync) {
1588 		if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1589 			const char *dst_name;
1590 
1591 			dst_name = trim_dotslash(buffer_pool_filename);
1592 			copy_file(ds_data, buffer_pool_filename, dst_name, 0);
1593 		}
1594 		if (file_exists("ib_lru_dump")) {
1595 			copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0);
1596 		}
1597 	}
1598 
1599 	if (has_rocksdb_plugin()) {
1600 		rocksdb_backup_checkpoint();
1601 	}
1602 
1603 	msg("Backup created in directory '%s'", xtrabackup_target_dir);
1604 	if (mysql_binlog_position != NULL) {
1605 		msg("MySQL binlog position: %s", mysql_binlog_position);
1606 	}
1607 	if (mysql_slave_position && opt_slave_info) {
1608 		msg("MySQL slave binlog position: %s",
1609 			mysql_slave_position);
1610 	}
1611 
1612 	if (!write_backup_config_file()) {
1613 		return(false);
1614 	}
1615 
1616 	if (!write_xtrabackup_info(mysql_connection, XTRABACKUP_INFO,
1617 				    opt_history != 0, true)) {
1618 		return(false);
1619 	}
1620 
1621 	return(true);
1622 }
1623 
1624 bool
ibx_copy_incremental_over_full()1625 ibx_copy_incremental_over_full()
1626 {
1627 	const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
1628 		"MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
1629 		NULL};
1630 	const char *sup_files[] = {"xtrabackup_binlog_info",
1631 				   "xtrabackup_galera_info",
1632 				   "xtrabackup_slave_info",
1633 				   "xtrabackup_info",
1634 				   "ib_lru_dump",
1635 				   NULL};
1636 	datadir_iter_t *it = NULL;
1637 	datadir_node_t node;
1638 	bool ret = true;
1639 	char path[FN_REFLEN];
1640 	int i;
1641 
1642 	datadir_node_init(&node);
1643 
1644 	/* If we were applying an incremental change set, we need to make
1645 	sure non-InnoDB files and xtrabackup_* metainfo files are copied
1646 	to the full backup directory. */
1647 
1648 	if (xtrabackup_incremental) {
1649 
1650 		ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL);
1651 
1652 		it = datadir_iter_new(xtrabackup_incremental_dir);
1653 
1654 		while (datadir_iter_next(it, &node)) {
1655 
1656 			/* copy only non-innodb files */
1657 
1658 			if (node.is_empty_dir
1659 			    || !filename_matches(node.filepath, ext_list)) {
1660 				continue;
1661 			}
1662 
1663 			if (file_exists(node.filepath_rel)) {
1664 				unlink(node.filepath_rel);
1665 			}
1666 
1667 			if (!(ret = copy_file(ds_data, node.filepath,
1668 						node.filepath_rel, 1))) {
1669 				msg("Failed to copy file %s",
1670 					node.filepath);
1671 				goto cleanup;
1672 			}
1673 		}
1674 
1675 		if (!(ret = backup_files_from_datadir(xtrabackup_incremental_dir)))
1676 			goto cleanup;
1677 
1678 		/* copy buffer pool dump */
1679 		if (innobase_buffer_pool_filename) {
1680 			const char *src_name;
1681 
1682 			src_name = trim_dotslash(innobase_buffer_pool_filename);
1683 
1684 			snprintf(path, sizeof(path), "%s/%s",
1685 				xtrabackup_incremental_dir,
1686 				src_name);
1687 
1688 			if (file_exists(path)) {
1689 				copy_file(ds_data, path,
1690 					  innobase_buffer_pool_filename, 0);
1691 			}
1692 		}
1693 
1694 		/* copy supplementary files */
1695 
1696 		for (i = 0; sup_files[i]; i++) {
1697 			snprintf(path, sizeof(path), "%s/%s",
1698 				xtrabackup_incremental_dir,
1699 				sup_files[i]);
1700 
1701 			if (file_exists(path))
1702 			{
1703 				if (file_exists(sup_files[i])) {
1704 					unlink(sup_files[i]);
1705 				}
1706 				copy_file(ds_data, path, sup_files[i], 0);
1707 			}
1708 		}
1709 
1710 		if (directory_exists(ROCKSDB_BACKUP_DIR, false)) {
1711 			if (my_rmtree(ROCKSDB_BACKUP_DIR, MYF(0))) {
1712 				die("Can't remove " ROCKSDB_BACKUP_DIR);
1713 			}
1714 		}
1715 		snprintf(path, sizeof(path), "%s/" ROCKSDB_BACKUP_DIR, xtrabackup_incremental_dir);
1716 		if (directory_exists(path, false)) {
1717 			if (my_mkdir(ROCKSDB_BACKUP_DIR, 0777, MYF(0))) {
1718 				die("my_mkdir failed for " ROCKSDB_BACKUP_DIR);
1719 			}
1720 			copy_or_move_dir(path, ROCKSDB_BACKUP_DIR, true, true);
1721 		}
1722 	}
1723 
1724 
1725 cleanup:
1726 	if (it != NULL) {
1727 		datadir_iter_free(it);
1728 	}
1729 
1730 	if (ds_data != NULL) {
1731 		ds_destroy(ds_data);
1732 	}
1733 
1734 	datadir_node_free(&node);
1735 
1736 	return(ret);
1737 }
1738 
1739 bool
ibx_cleanup_full_backup()1740 ibx_cleanup_full_backup()
1741 {
1742 	const char *ext_list[] = {"delta", "meta", "ibd", NULL};
1743 	datadir_iter_t *it = NULL;
1744 	datadir_node_t node;
1745 	bool ret = true;
1746 
1747 	datadir_node_init(&node);
1748 
1749 	/* If we are applying an incremental change set, we need to make
1750 	sure non-InnoDB files are cleaned up from full backup dir before
1751 	we copy files from incremental dir. */
1752 
1753 	it = datadir_iter_new(xtrabackup_target_dir);
1754 
1755 	while (datadir_iter_next(it, &node)) {
1756 
1757 		if (node.is_empty_dir) {
1758 #ifdef _WIN32
1759 			DeleteFile(node.filepath);
1760 #else
1761 			rmdir(node.filepath);
1762 #endif
1763 		}
1764 
1765 		if (xtrabackup_incremental && !node.is_empty_dir
1766 		    && !filename_matches(node.filepath, ext_list)) {
1767 			unlink(node.filepath);
1768 		}
1769 	}
1770 
1771 	datadir_iter_free(it);
1772 
1773 	datadir_node_free(&node);
1774 
1775 	return(ret);
1776 }
1777 
1778 bool
apply_log_finish()1779 apply_log_finish()
1780 {
1781 	if (!ibx_cleanup_full_backup()
1782 		|| !ibx_copy_incremental_over_full()) {
1783 		return(false);
1784 	}
1785 
1786 	return(true);
1787 }
1788 
1789 bool
copy_back()1790 copy_back()
1791 {
1792 	bool ret = false;
1793 	datadir_iter_t *it = NULL;
1794 	datadir_node_t node;
1795 	char *dst_dir;
1796 
1797 	memset(&node, 0, sizeof(node));
1798 
1799 	if (!opt_force_non_empty_dirs) {
1800 		if (!directory_exists_and_empty(mysql_data_home,
1801 							"Original data")) {
1802 			return(false);
1803 		}
1804 	} else {
1805 		if (!directory_exists(mysql_data_home, true)) {
1806 			return(false);
1807 		}
1808 	}
1809 	if (srv_undo_dir && *srv_undo_dir
1810 		&& !directory_exists(srv_undo_dir, true)) {
1811 			return(false);
1812 	}
1813 	if (innobase_data_home_dir && *innobase_data_home_dir
1814 		&& !directory_exists(innobase_data_home_dir, true)) {
1815 			return(false);
1816 	}
1817 	if (srv_log_group_home_dir && *srv_log_group_home_dir
1818 		&& !directory_exists(srv_log_group_home_dir, true)) {
1819 			return(false);
1820 	}
1821 
1822 	/* cd to backup directory */
1823 	if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
1824 	{
1825 		msg("Can't my_setwd %s", xtrabackup_target_dir);
1826 		return(false);
1827 	}
1828 
1829 	/* parse data file path */
1830 
1831 	if (!innobase_data_file_path) {
1832   		innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
1833 	}
1834 
1835 	srv_sys_space.set_path(".");
1836 
1837 	if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
1838 		msg("syntax error in innodb_data_file_path");
1839 		return(false);
1840 	}
1841 
1842 	srv_max_n_threads = 1000;
1843 	sync_check_init();
1844 	ut_crc32_init();
1845 
1846 	/* copy undo tablespaces */
1847 
1848 
1849 	dst_dir = (srv_undo_dir && *srv_undo_dir)
1850 			? srv_undo_dir : mysql_data_home;
1851 
1852 	ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1853 
1854 	for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) {
1855 		char filename[20];
1856 		sprintf(filename, "undo%03u", i);
1857 		if (!file_exists(filename)) {
1858 			break;
1859 		}
1860 		if (!(ret = copy_or_move_file(filename, filename,
1861 					      dst_dir, 1))) {
1862 			goto cleanup;
1863 		}
1864 	}
1865 
1866 	ds_destroy(ds_data);
1867 	ds_data = NULL;
1868 
1869 	/* copy redo logs */
1870 
1871 	dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir)
1872 		? srv_log_group_home_dir : mysql_data_home;
1873 
1874 	/* --backup generates a single ib_logfile0, which we must copy
1875 	if it exists. */
1876 
1877 	ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1878 	MY_STAT stat_arg;
1879 	if (!my_stat("ib_logfile0", &stat_arg, MYF(0)) || !stat_arg.st_size) {
1880 		/* After completed --prepare, redo log files are redundant.
1881 		We must delete any redo logs at the destination, so that
1882 		the database will not jump to a different log sequence number
1883 		(LSN). */
1884 
1885 		for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) {
1886 			char filename[FN_REFLEN];
1887 			snprintf(filename, sizeof filename, "%s/ib_logfile%u",
1888 				 dst_dir, i);
1889 			unlink(filename);
1890 		}
1891 	} else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0",
1892 					     dst_dir, 1))) {
1893 		goto cleanup;
1894 	}
1895 	ds_destroy(ds_data);
1896 
1897 	/* copy innodb system tablespace(s) */
1898 
1899 	dst_dir = (innobase_data_home_dir && *innobase_data_home_dir)
1900 				? innobase_data_home_dir : mysql_data_home;
1901 
1902 	ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1903 
1904 	for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1905 	     end(srv_sys_space.end());
1906 	     iter != end;
1907 	     ++iter) {
1908 		const char *filename = base_name(iter->name());
1909 
1910 		if (!(ret = copy_or_move_file(filename, iter->name(),
1911 					      dst_dir, 1))) {
1912 			goto cleanup;
1913 		}
1914 	}
1915 
1916 	ds_destroy(ds_data);
1917 
1918 	/* copy the rest of tablespaces */
1919 	ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL);
1920 
1921 	it = datadir_iter_new(".", false);
1922 
1923 	datadir_node_init(&node);
1924 
1925 	while (datadir_iter_next(it, &node)) {
1926 		const char *ext_list[] = {"backup-my.cnf",
1927 			"xtrabackup_binary", "xtrabackup_binlog_info",
1928 			"xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
1929 			NULL};
1930 		const char *filename;
1931 		char c_tmp;
1932 		int i_tmp;
1933 		bool is_ibdata_file;
1934 
1935 		if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
1936 #ifdef _WIN32
1937 			|| strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
1938 #endif
1939 		)
1940 		{
1941 			// copied at later step
1942 			continue;
1943 		}
1944 
1945 		/* create empty directories */
1946 		if (node.is_empty_dir) {
1947 			char path[FN_REFLEN];
1948 
1949 			snprintf(path, sizeof(path), "%s/%s",
1950 				mysql_data_home, node.filepath_rel);
1951 
1952 			msg("Creating directory %s", path);
1953 
1954 			if (mkdirp(path, 0777, MYF(0)) < 0) {
1955 				char errbuf[MYSYS_STRERROR_SIZE];
1956 				my_strerror(errbuf, sizeof(errbuf), my_errno);
1957 				msg("Can not create directory %s: %s",
1958 					path, errbuf);
1959 				ret = false;
1960 
1961 				goto cleanup;
1962 
1963 			}
1964 
1965 			msg(" ...done.");
1966 
1967 			continue;
1968 		}
1969 
1970 		filename = base_name(node.filepath);
1971 
1972 		/* skip .qp files */
1973 		if (filename_matches(filename, ext_list)) {
1974 			continue;
1975 		}
1976 
1977 		/* skip undo tablespaces */
1978 		if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) {
1979 			continue;
1980 		}
1981 
1982 		/* skip the redo log (it was already copied) */
1983 		if (!strcmp(filename, "ib_logfile0")) {
1984 			continue;
1985 		}
1986 
1987 		/* skip innodb data files */
1988 		is_ibdata_file = false;
1989 		for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1990 		       end(srv_sys_space.end()); iter != end; ++iter) {
1991 			const char *ibfile = base_name(iter->name());
1992 			if (strcmp(ibfile, filename) == 0) {
1993 				is_ibdata_file = true;
1994 				break;
1995 			}
1996 		}
1997 		if (is_ibdata_file) {
1998 			continue;
1999 		}
2000 
2001 		if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel,
2002 					      mysql_data_home, 1))) {
2003 			goto cleanup;
2004 		}
2005 	}
2006 
2007 	/* copy buffer pool dump */
2008 
2009 	if (innobase_buffer_pool_filename) {
2010 		const char *src_name;
2011 		char path[FN_REFLEN];
2012 
2013 		src_name = trim_dotslash(innobase_buffer_pool_filename);
2014 
2015 		snprintf(path, sizeof(path), "%s/%s",
2016 			mysql_data_home,
2017 			src_name);
2018 
2019 		/* could be already copied with other files
2020 		from data directory */
2021 		if (file_exists(src_name) &&
2022 			!file_exists(innobase_buffer_pool_filename)) {
2023 			copy_or_move_file(src_name,
2024 					  innobase_buffer_pool_filename,
2025 					  mysql_data_home, 0);
2026 		}
2027 	}
2028 
2029 	rocksdb_copy_back();
2030 
2031 cleanup:
2032 	if (it != NULL) {
2033 		datadir_iter_free(it);
2034 	}
2035 
2036 	datadir_node_free(&node);
2037 
2038 	if (ds_data != NULL) {
2039 		ds_destroy(ds_data);
2040 	}
2041 
2042 	ds_data = NULL;
2043 
2044 	sync_check_close();
2045 	return(ret);
2046 }
2047 
2048 bool
decrypt_decompress_file(const char * filepath,uint thread_n)2049 decrypt_decompress_file(const char *filepath, uint thread_n)
2050 {
2051 	std::stringstream cmd, message;
2052 	char *dest_filepath = strdup(filepath);
2053 	bool needs_action = false;
2054 
2055 	cmd << IF_WIN("type ","cat ") << filepath;
2056 
2057  	if (opt_decompress
2058  	    && ends_with(filepath, ".qp")) {
2059  		cmd << " | qpress -dio ";
2060  		dest_filepath[strlen(dest_filepath) - 3] = 0;
2061  		if (needs_action) {
2062  			message << " and ";
2063  		}
2064  		message << "decompressing";
2065  		needs_action = true;
2066  	}
2067 
2068  	cmd << " > " << dest_filepath;
2069  	message << " " << filepath;
2070 
2071  	free(dest_filepath);
2072 
2073  	if (needs_action) {
2074 
2075 		msg(thread_n,"%s\n", message.str().c_str());
2076 
2077 	 	if (system(cmd.str().c_str()) != 0) {
2078 	 		return(false);
2079 	 	}
2080 
2081 		if (opt_remove_original) {
2082 			msg(thread_n, "Removing %s", filepath);
2083 			if (my_delete(filepath, MYF(MY_WME)) != 0) {
2084 				return(false);
2085 			}
2086 		}
2087 	 }
2088 
2089  	return(true);
2090 }
2091 
2092 static
2093 os_thread_ret_t STDCALL
decrypt_decompress_thread_func(void * arg)2094 decrypt_decompress_thread_func(void *arg)
2095 {
2096 	bool ret = true;
2097 	datadir_node_t node;
2098 	datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg);
2099 
2100 	datadir_node_init(&node);
2101 
2102 	while (datadir_iter_next(ctxt->it, &node)) {
2103 
2104 		/* skip empty directories in backup */
2105 		if (node.is_empty_dir) {
2106 			continue;
2107 		}
2108 
2109 		if (!ends_with(node.filepath, ".qp")) {
2110 			continue;
2111 		}
2112 
2113 		if (!(ret = decrypt_decompress_file(node.filepath,
2114 							ctxt->n_thread))) {
2115 			goto cleanup;
2116 		}
2117 	}
2118 
2119 cleanup:
2120 
2121 	datadir_node_free(&node);
2122 
2123 	pthread_mutex_lock(ctxt->count_mutex);
2124 	--(*ctxt->count);
2125 	pthread_mutex_unlock(ctxt->count_mutex);
2126 
2127 	ctxt->ret = ret;
2128 
2129 	os_thread_exit();
2130 	OS_THREAD_DUMMY_RETURN;
2131 }
2132 
2133 bool
decrypt_decompress()2134 decrypt_decompress()
2135 {
2136 	bool ret;
2137 	datadir_iter_t *it = NULL;
2138 
2139 	srv_max_n_threads = 1000;
2140 	sync_check_init();
2141 
2142 	/* cd to backup directory */
2143 	if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
2144 	{
2145 		msg("Can't my_setwd %s", xtrabackup_target_dir);
2146 		return(false);
2147 	}
2148 
2149 	/* copy the rest of tablespaces */
2150 	ds_data = ds_create(".", DS_TYPE_LOCAL);
2151 
2152 	it = datadir_iter_new(".", false);
2153 
2154 	ut_a(xtrabackup_parallel >= 0);
2155 
2156 	ret = run_data_threads(it, decrypt_decompress_thread_func,
2157 		xtrabackup_parallel ? xtrabackup_parallel : 1);
2158 
2159 	if (it != NULL) {
2160 		datadir_iter_free(it);
2161 	}
2162 
2163 	if (ds_data != NULL) {
2164 		ds_destroy(ds_data);
2165 	}
2166 
2167 	ds_data = NULL;
2168 
2169 	sync_check_close();
2170 
2171 	return(ret);
2172 }
2173 
2174 /*
2175   Copy some files from top level datadir.
2176   Do not copy the Innodb files (ibdata1, redo log files),
2177   as this is done in a separate step.
2178 */
backup_files_from_datadir(const char * dir_path)2179 static bool backup_files_from_datadir(const char *dir_path)
2180 {
2181 	os_file_dir_t dir = os_file_opendir(dir_path);
2182 	if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false;
2183 
2184 	os_file_stat_t info;
2185 	bool ret = true;
2186 	while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
2187 
2188 		if (info.type != OS_FILE_TYPE_FILE)
2189 			continue;
2190 
2191 		const char *pname = strrchr(info.name, OS_PATH_SEPARATOR);
2192 		if (!pname)
2193 			pname = info.name;
2194 
2195 		if (!starts_with(pname, "aws-kms-key") &&
2196 			!starts_with(pname, "aria_log"))
2197 			/* For ES exchange the above line with the following code:
2198 			(!xtrabackup_prepare || !xtrabackup_incremental_dir ||
2199 				!starts_with(pname, "aria_log")))
2200 			*/
2201 			continue;
2202 
2203 		if (xtrabackup_prepare && xtrabackup_incremental_dir &&
2204 			file_exists(info.name))
2205 			unlink(info.name);
2206 
2207 		std::string full_path(dir_path);
2208 		full_path.append(1, OS_PATH_SEPARATOR).append(info.name);
2209 		if (!(ret = copy_file(ds_data, full_path.c_str() , info.name, 1)))
2210 			break;
2211 	}
2212 	os_file_closedir(dir);
2213 	return ret;
2214 }
2215 
2216 
rocksdb_remove_checkpoint_directory()2217 static int rocksdb_remove_checkpoint_directory()
2218 {
2219 	xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
2220 	return 0;
2221 }
2222 
has_rocksdb_plugin()2223 static bool has_rocksdb_plugin()
2224 {
2225 	static bool first_time = true;
2226 	static bool has_plugin= false;
2227 	if (!first_time || !xb_backup_rocksdb)
2228 		return has_plugin;
2229 
2230 	const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
2231 	MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
2232 	MYSQL_ROW row = mysql_fetch_row(result);
2233 	if (row)
2234 		has_plugin = !strcmp(row[0], "1");
2235 	mysql_free_result(result);
2236 	first_time = false;
2237 	return has_plugin;
2238 }
2239 
trim_trailing_dir_sep(char * path)2240 static char *trim_trailing_dir_sep(char *path)
2241 {
2242 	size_t path_len = strlen(path);
2243 	while (path_len)
2244 	{
2245 		char c = path[path_len - 1];
2246 		if (c == '/' IF_WIN(|| c == '\\', ))
2247 			path_len--;
2248 		else
2249 			break;
2250 	}
2251 	path[path_len] = 0;
2252 	return path;
2253 }
2254 
2255 /*
2256 Create a file hardlink.
2257 @return true on success, false on error.
2258 */
make_hardlink(const char * from_path,const char * to_path)2259 static bool make_hardlink(const char *from_path, const char *to_path)
2260 {
2261 	DBUG_EXECUTE_IF("no_hardlinks", return false;);
2262 	char to_path_full[FN_REFLEN];
2263 	if (!is_abs_path(to_path))
2264 	{
2265 		fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
2266 	}
2267 	else
2268 	{
2269 		strncpy(to_path_full, to_path, sizeof(to_path_full));
2270 	}
2271 #ifdef _WIN32
2272 	return  CreateHardLink(to_path_full, from_path, NULL);
2273 #else
2274 	return !link(from_path, to_path_full);
2275 #endif
2276 }
2277 
2278 /*
2279  Copies or moves a directory (non-recursively so far).
2280  Helper function used to backup rocksdb checkpoint, or copy-back the
2281  rocksdb files.
2282 
2283  Has optimization that allows to use hardlinks when possible
2284  (source and destination are directories on the same device)
2285 */
copy_or_move_dir(const char * from,const char * to,bool do_copy,bool allow_hardlinks)2286 static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
2287 {
2288 	datadir_node_t node;
2289 	datadir_node_init(&node);
2290 	datadir_iter_t *it = datadir_iter_new(from, false);
2291 
2292 	while (datadir_iter_next(it, &node))
2293 	{
2294 		char to_path[FN_REFLEN];
2295 		const char *from_path = node.filepath;
2296 		snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
2297 		bool rc = false;
2298 		if (do_copy && allow_hardlinks)
2299 		{
2300 			rc = make_hardlink(from_path, to_path);
2301 			if (rc)
2302 			{
2303 				msg("Creating hardlink from %s to %s",from_path, to_path);
2304 			}
2305 			else
2306 			{
2307 				allow_hardlinks = false;
2308 			}
2309 		}
2310 
2311 		if (!rc)
2312 		{
2313 			rc = (do_copy ?
2314 				copy_file(ds_data, from_path, to_path, 1) :
2315 				move_file(ds_data, from_path, node.filepath_rel,
2316 					to, 1));
2317 		}
2318 		if (!rc)
2319 			die("copy or move file failed");
2320 	}
2321 	datadir_iter_free(it);
2322 	datadir_node_free(&node);
2323 
2324 }
2325 
2326 /*
2327   Obtain user level lock , to protect the checkpoint directory of the server
2328   from being  user/overwritten by different backup processes, if backups are
2329   running in parallel.
2330 
2331   This lock will be acquired before rocksdb checkpoint is created,  held
2332   while all files from it are being copied to their final backup destination,
2333   and finally released after the checkpoint is removed.
2334 */
rocksdb_lock_checkpoint()2335 static void rocksdb_lock_checkpoint()
2336 {
2337 	msg("Obtaining rocksdb checkpoint lock.");
2338 	MYSQL_RES *res =
2339 		xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);
2340 
2341 	MYSQL_ROW r = mysql_fetch_row(res);
2342 	if (r && r[0] && strcmp(r[0], "1"))
2343 	{
2344 		msg("Could not obtain rocksdb checkpont lock.");
2345 		exit(EXIT_FAILURE);
2346 	}
2347 	mysql_free_result(res);
2348 }
2349 
rocksdb_unlock_checkpoint()2350 static void rocksdb_unlock_checkpoint()
2351 {
2352 	xb_mysql_query(mysql_connection,
2353 		"SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
2354 }
2355 
2356 
2357 /*
2358   Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
2359   directory.
2360   A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
2361   acquired be this function.
2362 */
2363 #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
2364 static 	char rocksdb_checkpoint_dir[FN_REFLEN];
2365 
rocksdb_create_checkpoint()2366 static void rocksdb_create_checkpoint()
2367 {
2368 	MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
2369 	MYSQL_ROW row = mysql_fetch_row(result);
2370 
2371 	DBUG_ASSERT(row && row[0] && row[1]);
2372 
2373 	char *rocksdbdir = row[0];
2374 	char *datadir = row[1];
2375 
2376 	if (is_abs_path(rocksdbdir))
2377 	{
2378 		snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
2379 			"%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
2380 	}
2381 	else
2382 	{
2383 		snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
2384 			"%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
2385 			trim_dotslash(rocksdbdir));
2386 	}
2387 	mysql_free_result(result);
2388 
2389 #ifdef _WIN32
2390 	for (char *p = rocksdb_checkpoint_dir; *p; p++)
2391 		if (*p == '\\') *p = '/';
2392 #endif
2393 
2394 	rocksdb_lock_checkpoint();
2395 
2396 	if (!access(rocksdb_checkpoint_dir, 0))
2397 	{
2398 		msg("Removing rocksdb checkpoint from previous backup attempt.");
2399 		rocksdb_remove_checkpoint_directory();
2400 	}
2401 
2402 	char query[FN_REFLEN + 32];
2403 	snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
2404 	xb_mysql_query(mysql_connection, query, false, true);
2405 }
2406 
2407 /*
2408   Copy files from rocksdb temporary checkpoint to final destination.
2409   remove temp.checkpoint directory (in server's datadir)
2410   and release user level lock acquired inside rocksdb_create_checkpoint().
2411 */
rocksdb_backup_checkpoint()2412 static void rocksdb_backup_checkpoint()
2413 {
2414 	msg("Backing up rocksdb files.");
2415 	char rocksdb_backup_dir[FN_REFLEN];
2416 	snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
2417 	bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
2418 	if (backup_to_directory)
2419 	{
2420 		if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
2421 			die("Can't create rocksdb backup directory %s", rocksdb_backup_dir);
2422 		}
2423 	}
2424 	copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
2425 	rocksdb_remove_checkpoint_directory();
2426 	rocksdb_unlock_checkpoint();
2427 }
2428 
2429 /*
2430   Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
2431 */
rocksdb_copy_back()2432 static void rocksdb_copy_back() {
2433 	if (access(ROCKSDB_BACKUP_DIR, 0))
2434 		return;
2435 	char rocksdb_home_dir[FN_REFLEN];
2436         if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
2437 		strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof rocksdb_home_dir - 1);
2438 		rocksdb_home_dir[sizeof rocksdb_home_dir - 1] = '\0';
2439 	} else {
2440 	   snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home,
2441 		xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
2442 	}
2443 	mkdirp(rocksdb_home_dir, 0777, MYF(0));
2444 	copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
2445 }
2446