1 /******************************************************
2 MariaBackup: hot backup tool for InnoDB
3 (c) 2009-2017 Percona LLC and/or its affiliates
4 Originally Created 3/3/2009 Yasufumi Kinoshita
5 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
6 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
7 (c) 2017, 2021, MariaDB Corporation.
8 Portions written by Marko Mäkelä.
9
10 This program is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; version 2 of the License.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
22
23 *******************************************************
24
25 This file incorporates work covered by the following copyright and
26 permission notice:
27
28 Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
29
30 This program is free software; you can redistribute it and/or modify it under
31 the terms of the GNU General Public License as published by the Free Software
32 Foundation; version 2 of the License.
33
34 This program is distributed in the hope that it will be useful, but WITHOUT
35 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
36 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
37
38 You should have received a copy of the GNU General Public License along with
39 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
40 Street, Fifth Floor, Boston, MA 02110-1335 USA
41
42 *******************************************************/
43
44 //#define XTRABACKUP_TARGET_IS_PLUGIN
45
46 #include <my_global.h>
47 #include <my_config.h>
48 #include <unireg.h>
49 #include <mysql_version.h>
50 #include <my_base.h>
51 #include <my_getopt.h>
52 #include <mysql_com.h>
53 #include <my_default.h>
54 #include <sql_class.h>
55
56 #include <fcntl.h>
57 #include <string.h>
58
59 #ifdef __linux__
60 # include <sys/prctl.h>
61 #include <sys/resource.h>
62 #endif
63
64 #ifdef __APPLE__
65 # include "libproc.h"
66 #endif
67
68
69 #include <btr0sea.h>
70 #include <dict0priv.h>
71 #include <lock0lock.h>
72 #include <log0recv.h>
73 #include <log0crypt.h>
74 #include <row0mysql.h>
75 #include <row0quiesce.h>
76 #include <srv0start.h>
77 #include "trx0sys.h"
78 #include <buf0dblwr.h>
79 #include "ha_innodb.h"
80
81 #include <list>
82 #include <sstream>
83 #include <set>
84 #include <fstream>
85 #include <mysql.h>
86
87 #define G_PTR uchar*
88
89 #include "common.h"
90 #include "datasink.h"
91
92 #include "xb_regex.h"
93 #include "fil_cur.h"
94 #include "write_filt.h"
95 #include "xtrabackup.h"
96 #include "ds_buffer.h"
97 #include "ds_tmpfile.h"
98 #include "xbstream.h"
99 #include "changed_page_bitmap.h"
100 #include "read_filt.h"
101 #include "backup_wsrep.h"
102 #include "innobackupex.h"
103 #include "backup_mysql.h"
104 #include "backup_copy.h"
105 #include "backup_mysql.h"
106 #include "encryption_plugin.h"
107 #include <sql_plugin.h>
108 #include <srv0srv.h>
109 #include <crc_glue.h>
110 #include <log.h>
111 #include <derror.h>
112 #include "backup_debug.h"
113
114 #define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages"
115
116 int sys_var_init();
117
118 /* === xtrabackup specific options === */
119 char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/";
120 char *xtrabackup_target_dir= xtrabackup_real_target_dir;
121 static my_bool xtrabackup_version;
122 static my_bool verbose;
123 my_bool xtrabackup_backup;
124 my_bool xtrabackup_prepare;
125 my_bool xtrabackup_copy_back;
126 my_bool xtrabackup_move_back;
127 my_bool xtrabackup_decrypt_decompress;
128 my_bool xtrabackup_print_param;
129 my_bool xtrabackup_mysqld_args;
130 my_bool xtrabackup_help;
131
132 my_bool xtrabackup_export;
133
134 my_bool xtrabackup_rollback_xa;
135
136 longlong xtrabackup_use_memory;
137
138 uint opt_protocol;
139 long xtrabackup_throttle; /* 0:unlimited */
140 static lint io_ticket;
141 static os_event_t wait_throttle;
142 static os_event_t log_copying_stop;
143
144 char *xtrabackup_incremental;
145 lsn_t incremental_lsn;
146 lsn_t incremental_to_lsn;
147 lsn_t incremental_last_lsn;
148 xb_page_bitmap *changed_page_bitmap;
149
150 char *xtrabackup_incremental_basedir; /* for --backup */
151 char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */
152 char *xtrabackup_incremental_dir; /* for --prepare */
153
154 char xtrabackup_real_incremental_basedir[FN_REFLEN];
155 char xtrabackup_real_extra_lsndir[FN_REFLEN];
156 char xtrabackup_real_incremental_dir[FN_REFLEN];
157
158
159 char *xtrabackup_tmpdir;
160
161 char *xtrabackup_tables;
162 char *xtrabackup_tables_file;
163 char *xtrabackup_tables_exclude;
164 char *xb_rocksdb_datadir;
165 my_bool xb_backup_rocksdb = 1;
166
167 typedef std::list<regex_t> regex_list_t;
168 static regex_list_t regex_include_list;
169 static regex_list_t regex_exclude_list;
170
171 static hash_table_t* tables_include_hash = NULL;
172 static hash_table_t* tables_exclude_hash = NULL;
173
174 char *xtrabackup_databases = NULL;
175 char *xtrabackup_databases_file = NULL;
176 char *xtrabackup_databases_exclude = NULL;
177 static hash_table_t* databases_include_hash = NULL;
178 static hash_table_t* databases_exclude_hash = NULL;
179
180 static hash_table_t* inc_dir_tables_hash;
181
182 struct xb_filter_entry_struct{
183 char* name;
184 ibool has_tables;
185 hash_node_t name_hash;
186 };
187 typedef struct xb_filter_entry_struct xb_filter_entry_t;
188
189 lsn_t checkpoint_lsn_start;
190 lsn_t checkpoint_no_start;
191 static lsn_t log_copy_scanned_lsn;
192 static bool log_copying_running;
193 static bool io_watching_thread_running;
194
195 int xtrabackup_parallel;
196
197 char *xtrabackup_stream_str = NULL;
198 xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE;
199 ibool xtrabackup_stream = FALSE;
200
201 const char *xtrabackup_compress_alg = NULL;
202 uint xtrabackup_compress = FALSE;
203 uint xtrabackup_compress_threads;
204 ulonglong xtrabackup_compress_chunk_size = 0;
205
206 /* sleep interval beetween log copy iterations in log copying thread
207 in milliseconds (default is 1 second) */
208 ulint xtrabackup_log_copy_interval = 1000;
209 static ulong max_buf_pool_modified_pct;
210
211 /* Ignored option (--log) for MySQL option compatibility */
212 static char* log_ignored_opt;
213
214
215 extern my_bool opt_use_ssl;
216 extern char *opt_tls_version;
217 my_bool opt_ssl_verify_server_cert;
218 my_bool opt_extended_validation;
219 my_bool opt_encrypted_backup;
220
221 /* === metadata of backup === */
222 #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
223 char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
224 static lsn_t metadata_from_lsn;
225 lsn_t metadata_to_lsn;
226 static lsn_t metadata_last_lsn;
227
228 static ds_file_t* dst_log_file;
229
230 static char mysql_data_home_buff[2];
231
232 const char *defaults_group = "mysqld";
233
234 /* === static parameters in ha_innodb.cc */
235
236 #define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */
237 #define HA_INNOBASE_RANGE_COUNT 100
238
239 /* The default values for the following, type long or longlong, start-up
240 parameters are declared in mysqld.cc: */
241
242 long innobase_buffer_pool_awe_mem_mb = 0;
243 long innobase_file_io_threads = 4;
244 long innobase_read_io_threads = 4;
245 long innobase_write_io_threads = 4;
246
247 longlong innobase_page_size = (1LL << 14); /* 16KB */
248 char* innobase_buffer_pool_filename = NULL;
249
250 /* The default values for the following char* start-up parameters
251 are determined in innobase_init below: */
252
253 static char* innobase_ignored_opt;
254 char* innobase_data_home_dir;
255 char* innobase_data_file_path;
256
257 my_bool innobase_use_doublewrite;
258 my_bool innobase_file_per_table;
259 my_bool innobase_locks_unsafe_for_binlog;
260 my_bool innobase_rollback_on_timeout;
261 my_bool innobase_create_status_file;
262
263 /* The following counter is used to convey information to InnoDB
264 about server activity: in selects it is not sensible to call
265 srv_active_wake_master_thread after each fetch or search, we only do
266 it every INNOBASE_WAKE_INTERVAL'th step. */
267
268 #define INNOBASE_WAKE_INTERVAL 32
269 ulong innobase_active_counter = 0;
270
271
272 my_bool xtrabackup_incremental_force_scan = FALSE;
273
274 /*
275 * Ignore corrupt pages (disabled by default; used
276 * by "innobackupex" as a command line argument).
277 */
278 ulong xtrabackup_innodb_force_recovery = 0;
279
280 /* The flushed lsn which is read from data files */
281 lsn_t flushed_lsn= 0;
282
283 ulong xb_open_files_limit= 0;
284 char *xb_plugin_dir;
285 char *xb_plugin_load;
286 my_bool xb_close_files;
287
288 /* Datasinks */
289 ds_ctxt_t *ds_data = NULL;
290 ds_ctxt_t *ds_meta = NULL;
291 ds_ctxt_t *ds_redo = NULL;
292
293 static bool innobackupex_mode = false;
294
295 /* String buffer used by --print-param to accumulate server options as they are
296 parsed from the defaults file */
297 static std::ostringstream print_param_str;
298
299 /* Set of specified parameters */
300 std::set<std::string> param_set;
301
302 static ulonglong global_max_value;
303
304 extern "C" sig_handler handle_fatal_signal(int sig);
305 extern LOGGER logger;
306
307 my_bool opt_galera_info = FALSE;
308 my_bool opt_slave_info = FALSE;
309 my_bool opt_no_lock = FALSE;
310 my_bool opt_safe_slave_backup = FALSE;
311 my_bool opt_rsync = FALSE;
312 my_bool opt_force_non_empty_dirs = FALSE;
313 my_bool opt_noversioncheck = FALSE;
314 my_bool opt_no_backup_locks = FALSE;
315 my_bool opt_decompress = FALSE;
316 my_bool opt_remove_original;
317 my_bool opt_log_innodb_page_corruption;
318
319 my_bool opt_lock_ddl_per_table = FALSE;
320 static my_bool opt_check_privileges;
321
322 extern const char *innodb_checksum_algorithm_names[];
323 extern TYPELIB innodb_checksum_algorithm_typelib;
324 extern const char *innodb_flush_method_names[];
325 extern TYPELIB innodb_flush_method_typelib;
326
327 static const char *binlog_info_values[] = {"off", "lockless", "on", "auto",
328 NullS};
329 static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "",
330 binlog_info_values, NULL};
331 ulong opt_binlog_info;
332
333 char *opt_incremental_history_name;
334 char *opt_incremental_history_uuid;
335
336 char *opt_user;
337 char *opt_password;
338 char *opt_host;
339 char *opt_defaults_group;
340 char *opt_socket;
341 uint opt_port;
342 char *opt_log_bin;
343
344 const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS};
345
346 TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "",
347 query_type_names, NULL};
348
349 ulong opt_lock_wait_query_type;
350 ulong opt_kill_long_query_type;
351
352 uint opt_kill_long_queries_timeout = 0;
353 uint opt_lock_wait_timeout = 0;
354 uint opt_lock_wait_threshold = 0;
355 uint opt_debug_sleep_before_unlock = 0;
356 uint opt_safe_slave_backup_timeout = 0;
357
358 const char *opt_history = NULL;
359
360
361 char mariabackup_exe[FN_REFLEN];
362 char orig_argv1[FN_REFLEN];
363
364 pthread_mutex_t backup_mutex;
365 pthread_cond_t scanned_lsn_cond;
366
367 typedef std::map<space_id_t,std::string> space_id_to_name_t;
368
369 struct ddl_tracker_t {
370 /** Tablspaces with their ID and name, as they were copied to backup.*/
371 space_id_to_name_t tables_in_backup;
372 /** Tablespaces for that optimized DDL without redo log was found.*/
373 std::set<space_id_t> optimized_ddl;
374 /** Drop operations found in redo log. */
375 std::set<space_id_t> drops;
376 /* For DDL operation found in redo log, */
377 space_id_to_name_t id_to_name;
378 };
379
380 static ddl_tracker_t ddl_tracker;
381
382 // Convert non-null terminated filename to space name
383 std::string filename_to_spacename(const byte *filename, size_t len);
384
CorruptedPages()385 CorruptedPages::CorruptedPages() { ut_a(!pthread_mutex_init(&m_mutex, NULL)); }
386
~CorruptedPages()387 CorruptedPages::~CorruptedPages() { ut_a(!pthread_mutex_destroy(&m_mutex)); }
388
add_page_no_lock(const char * space_name,ulint space_id,ulint page_no,bool convert_space_name)389 void CorruptedPages::add_page_no_lock(const char *space_name, ulint space_id,
390 ulint page_no, bool convert_space_name)
391 {
392 space_info_t &space_info = m_spaces[space_id];
393 if (space_info.space_name.empty())
394 space_info.space_name=
395 convert_space_name
396 ? filename_to_spacename(reinterpret_cast<const byte *>(space_name),
397 strlen(space_name))
398 : space_name;
399 (void)space_info.pages.insert(page_no);
400 }
401
add_page(const char * file_name,ulint space_id,ulint page_no)402 void CorruptedPages::add_page(const char *file_name, ulint space_id,
403 ulint page_no)
404 {
405 ut_a(!pthread_mutex_lock(&m_mutex));
406 add_page_no_lock(file_name, space_id, page_no, true);
407 ut_a(!pthread_mutex_unlock(&m_mutex));
408 }
409
contains(ulint space_id,ulint page_no) const410 bool CorruptedPages::contains(ulint space_id, ulint page_no) const
411 {
412 bool result = false;
413 ut_a(!pthread_mutex_lock(&m_mutex));
414 container_t::const_iterator space_it= m_spaces.find(space_id);
415 if (space_it != m_spaces.end())
416 result = space_it->second.pages.count(page_no);
417 ut_a(!pthread_mutex_unlock(&m_mutex));
418 return result;
419 }
420
drop_space(ulint space_id)421 void CorruptedPages::drop_space(ulint space_id)
422 {
423 ut_a(!pthread_mutex_lock(&m_mutex));
424 m_spaces.erase(space_id);
425 ut_a(!pthread_mutex_unlock(&m_mutex));
426 }
427
rename_space(ulint space_id,const std::string & new_name)428 void CorruptedPages::rename_space(ulint space_id, const std::string &new_name)
429 {
430 ut_a(!pthread_mutex_lock(&m_mutex));
431 container_t::iterator space_it = m_spaces.find(space_id);
432 if (space_it != m_spaces.end())
433 space_it->second.space_name = new_name;
434 ut_a(!pthread_mutex_unlock(&m_mutex));
435 }
436
print_to_file(const char * filename) const437 bool CorruptedPages::print_to_file(const char *filename) const
438 {
439 std::ostringstream out;
440 ut_a(!pthread_mutex_lock(&m_mutex));
441 if (!m_spaces.size())
442 {
443 ut_a(!pthread_mutex_unlock(&m_mutex));
444 return true;
445 }
446 for (container_t::const_iterator space_it=
447 m_spaces.begin();
448 space_it != m_spaces.end(); ++space_it)
449 {
450 out << space_it->second.space_name << " " << space_it->first << "\n";
451 bool first_page_no= true;
452 for (std::set<ulint>::const_iterator page_it=
453 space_it->second.pages.begin();
454 page_it != space_it->second.pages.end(); ++page_it)
455 if (first_page_no)
456 {
457 out << *page_it;
458 first_page_no= false;
459 }
460 else
461 out << " " << *page_it;
462 out << "\n";
463 }
464 ut_a(!pthread_mutex_unlock(&m_mutex));
465 if (xtrabackup_backup)
466 return backup_file_print_buf(filename, out.str().c_str(),
467 static_cast<int>(out.str().size()));
468 std::ofstream outfile;
469 outfile.open(filename);
470 if (!outfile.is_open())
471 die("Can't open %s, error number: %d, error message: %s", filename, errno,
472 strerror(errno));
473 outfile << out.str();
474 return true;
475 }
476
read_from_file(const char * file_name)477 void CorruptedPages::read_from_file(const char *file_name)
478 {
479 MY_STAT mystat;
480 if (!my_stat(file_name, &mystat, MYF(0)))
481 return;
482 std::ifstream infile;
483 infile.open(file_name);
484 if (!infile.is_open())
485 die("Can't open %s, error number: %d, error message: %s", file_name, errno,
486 strerror(errno));
487 std::string line;
488 std::string space_name;
489 ulint space_id;
490 ulint line_number= 0;
491 while (std::getline(infile, line))
492 {
493 ++line_number;
494 std::istringstream iss(line);
495 if (line_number & 1) {
496 if (!(iss >> space_name))
497 die("Can't parse space name from corrupted pages file at "
498 "line " ULINTPF,
499 line_number);
500 if (!(iss >> space_id))
501 die("Can't parse space id from corrupted pages file at line " ULINTPF,
502 line_number);
503 }
504 else
505 {
506 ulint page_no;
507 while ((iss >> page_no))
508 add_page_no_lock(space_name.c_str(), space_id, page_no, false);
509 if (!iss.eof())
510 die("Corrupted pages file parse error on line number " ULINTPF,
511 line_number);
512 }
513 }
514 }
515
empty() const516 bool CorruptedPages::empty() const
517 {
518 ut_a(!pthread_mutex_lock(&m_mutex));
519 bool result= !m_spaces.size();
520 ut_a(!pthread_mutex_unlock(&m_mutex));
521 return result;
522 }
523
524 static void xb_load_single_table_tablespace(const std::string &space_name,
525 bool set_size);
526 static void xb_data_files_close();
527 static fil_space_t* fil_space_get_by_name(const char* name);
528
zero_out_free_pages()529 void CorruptedPages::zero_out_free_pages()
530 {
531 container_t non_free_pages;
532 byte* buf= static_cast<byte*>(ut_malloc_nokey(2 * srv_page_size));
533 byte* zero_page = static_cast<byte*>(ut_align(buf, srv_page_size));
534 memset(zero_page, 0, srv_page_size);
535
536 ut_a(!pthread_mutex_lock(&m_mutex));
537 for (container_t::const_iterator space_it= m_spaces.begin();
538 space_it != m_spaces.end(); ++space_it)
539 {
540 ulint space_id = space_it->first;
541 const std::string &space_name = space_it->second.space_name;
542 // There is no need to close tablespaces explixitly as they will be closed
543 // in innodb_shutdown().
544 xb_load_single_table_tablespace(space_name, false);
545 mutex_enter(&fil_system.mutex);
546 fil_space_t *space = fil_space_get_by_name(space_name.c_str());
547 mutex_exit(&fil_system.mutex);
548 if (!space)
549 die("Can't find space object for space name %s to check corrupted page",
550 space_name.c_str());
551 for (std::set<ulint>::const_iterator page_it=
552 space_it->second.pages.begin();
553 page_it != space_it->second.pages.end(); ++page_it)
554 {
555 bool is_free= fseg_page_is_free(space, static_cast<unsigned>(*page_it));
556 if (!is_free) {
557 space_info_t &space_info = non_free_pages[space_id];
558 space_info.pages.insert(*page_it);
559 if (space_info.space_name.empty())
560 space_info.space_name = space_name;
561 msg("Error: corrupted page " ULINTPF
562 " of tablespace %s can not be fixed",
563 *page_it, space_name.c_str());
564 }
565 else
566 {
567 const page_id_t page_id(space->id, *page_it);
568 dberr_t err= fil_io(IORequestWrite, true, page_id, 0, 0,
569 srv_page_size, zero_page, NULL);
570 if (err != DB_SUCCESS)
571 die("Can't zero out corrupted page " ULINTPF " of tablespace %s",
572 *page_it, space_name.c_str());
573 msg("Corrupted page " ULINTPF
574 " of tablespace %s was successfuly fixed.",
575 *page_it, space_name.c_str());
576 }
577 }
578 }
579 m_spaces.swap(non_free_pages);
580 ut_a(!pthread_mutex_unlock(&m_mutex));
581 ut_free(buf);
582 }
583
584 /* Simple datasink creation tracking...add datasinks in the reverse order you
585 want them destroyed. */
586 #define XTRABACKUP_MAX_DATASINKS 10
587 static ds_ctxt_t *datasinks[XTRABACKUP_MAX_DATASINKS];
588 static uint actual_datasinks = 0;
589 static inline
590 void
xtrabackup_add_datasink(ds_ctxt_t * ds)591 xtrabackup_add_datasink(ds_ctxt_t *ds)
592 {
593 xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS);
594 datasinks[actual_datasinks] = ds; actual_datasinks++;
595 }
596
597 typedef void (*process_single_tablespace_func_t)(const char *dirname,
598 const char *filname,
599 bool is_remote,
600 bool set_size);
601 static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback);
602
603 /* ======== Datafiles iterator ======== */
604 struct datafiles_iter_t {
605 fil_space_t *space;
606 fil_node_t *node;
607 ibool started;
608 pthread_mutex_t mutex;
609 };
610
611 /* ======== Datafiles iterator ======== */
612 static
613 datafiles_iter_t *
datafiles_iter_new()614 datafiles_iter_new()
615 {
616 datafiles_iter_t *it;
617
618 it = static_cast<datafiles_iter_t *>(malloc(sizeof(datafiles_iter_t)));
619 pthread_mutex_init(&it->mutex, NULL);
620
621 it->space = NULL;
622 it->node = NULL;
623 it->started = FALSE;
624
625 return it;
626 }
627
628 static
629 fil_node_t *
datafiles_iter_next(datafiles_iter_t * it)630 datafiles_iter_next(datafiles_iter_t *it)
631 {
632 fil_node_t *new_node;
633
634 pthread_mutex_lock(&it->mutex);
635
636 if (it->node == NULL) {
637 if (it->started)
638 goto end;
639 it->started = TRUE;
640 } else {
641 it->node = UT_LIST_GET_NEXT(chain, it->node);
642 if (it->node != NULL)
643 goto end;
644 }
645
646 it->space = (it->space == NULL) ?
647 UT_LIST_GET_FIRST(fil_system.space_list) :
648 UT_LIST_GET_NEXT(space_list, it->space);
649
650 while (it->space != NULL &&
651 (it->space->purpose != FIL_TYPE_TABLESPACE ||
652 UT_LIST_GET_LEN(it->space->chain) == 0))
653 it->space = UT_LIST_GET_NEXT(space_list, it->space);
654 if (it->space == NULL)
655 goto end;
656
657 it->node = UT_LIST_GET_FIRST(it->space->chain);
658
659 end:
660 new_node = it->node;
661 pthread_mutex_unlock(&it->mutex);
662
663 return new_node;
664 }
665
666 static
667 void
datafiles_iter_free(datafiles_iter_t * it)668 datafiles_iter_free(datafiles_iter_t *it)
669 {
670 pthread_mutex_destroy(&it->mutex);
671 free(it);
672 }
673
674 #ifndef DBUG_OFF
675 struct dbug_thread_param_t
676 {
677 MYSQL *con;
678 const char *query;
679 int expect_err;
680 int expect_errno;
681 os_event_t done_event;
682 };
683
684
685 /* Thread procedure used in dbug_start_query_thread. */
686 extern "C"
687 os_thread_ret_t
DECLARE_THREAD(dbug_execute_in_new_connection)688 DECLARE_THREAD(dbug_execute_in_new_connection)(void *arg)
689 {
690 mysql_thread_init();
691 dbug_thread_param_t *par= (dbug_thread_param_t *)arg;
692 int err = mysql_query(par->con, par->query);
693 int err_no = mysql_errno(par->con);
694 if(par->expect_err != err)
695 {
696 msg("FATAL: dbug_execute_in_new_connection : mysql_query '%s' returns %d, instead of expected %d",
697 par->query, err, par->expect_err);
698 _exit(1);
699 }
700 if (err && par->expect_errno && par->expect_errno != err_no)
701 {
702 msg("FATAL: dbug_execute_in_new_connection: mysql_query '%s' returns mysql_errno %d, instead of expected %d",
703 par->query, err_no, par->expect_errno);
704 _exit(1);
705 }
706 mysql_close(par->con);
707 mysql_thread_end();
708 os_event_t done = par->done_event;
709 delete par;
710 os_event_set(done);
711 os_thread_exit();
712 return os_thread_ret_t(0);
713 }
714
715 /*
716 Execute query from a new connection, in own thread.
717
718 @param query - query to be executed
719 @param wait_state - if not NULL, wait until query from new connection
720 reaches this state (value of column State in I_S.PROCESSLIST)
721 @param expected_err - if 0, query is supposed to finish successfully,
722 otherwise query should return error.
723 @param expected_errno - if not 0, and query finished with error,
724 expected mysql_errno()
725 */
dbug_start_query_thread(const char * query,const char * wait_state,int expected_err,int expected_errno)726 static os_event_t dbug_start_query_thread(
727 const char *query,
728 const char *wait_state,
729 int expected_err,
730 int expected_errno)
731
732 {
733 dbug_thread_param_t *par = new dbug_thread_param_t;
734 par->query = query;
735 par->expect_err = expected_err;
736 par->expect_errno = expected_errno;
737 par->done_event = os_event_create(0);
738 par->con = xb_mysql_connect();
739 os_thread_create(dbug_execute_in_new_connection, par, 0);
740
741 if (!wait_state)
742 return par->done_event;
743
744 char q[256];
745 snprintf(q, sizeof(q),
746 "SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST where ID=%lu"
747 " AND Command='Query' AND State='%s'",
748 mysql_thread_id(par->con), wait_state);
749 for (;;) {
750 MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true);
751 bool exists = mysql_fetch_row(result) != NULL;
752 mysql_free_result(result);
753 if (exists) {
754 goto end;
755 }
756 msg("Waiting for query '%s' on connection %lu to "
757 " reach state '%s'", query, mysql_thread_id(par->con),
758 wait_state);
759 my_sleep(1000);
760 }
761 end:
762 msg("query '%s' on connection %lu reached state '%s'", query,
763 mysql_thread_id(par->con), wait_state);
764 return par->done_event;
765 }
766
767 os_event_t dbug_alter_thread_done;
768 #endif
769
mdl_lock_all()770 void mdl_lock_all()
771 {
772 mdl_lock_init();
773 datafiles_iter_t *it = datafiles_iter_new();
774 if (!it)
775 return;
776
777 while (fil_node_t *node = datafiles_iter_next(it)){
778 if (fil_is_user_tablespace_id(node->space->id)
779 && check_if_skip_table(node->space->name))
780 continue;
781
782 mdl_lock_table(node->space->id);
783 }
784 datafiles_iter_free(it);
785 }
786
787
788 // Convert non-null terminated filename to space name
filename_to_spacename(const byte * filename,size_t len)789 std::string filename_to_spacename(const byte *filename, size_t len)
790 {
791 // null- terminate filename
792 char *f = (char *)malloc(len + 1);
793 ut_a(f);
794 memcpy(f, filename, len);
795 f[len] = 0;
796 for (size_t i = 0; i < len; i++)
797 if (f[i] == '\\')
798 f[i] = '/';
799 char *p = strrchr(f, '.');
800 ut_a(p);
801 *p = 0;
802 char *table = strrchr(f, '/');
803 ut_a(table);
804 *table = 0;
805 char *db = strrchr(f, '/');
806 ut_a(db);
807 *table = '/';
808 std::string s(db+1);
809 free(f);
810 return s;
811 }
812
813 /** Report an operation to create, delete, or rename a file during backup.
814 @param[in] space_id tablespace identifier
815 @param[in] flags tablespace flags (NULL if not create)
816 @param[in] name file name (not NUL-terminated)
817 @param[in] len length of name, in bytes
818 @param[in] new_name new file name (NULL if not rename)
819 @param[in] new_len length of new_name, in bytes (0 if NULL) */
backup_file_op(ulint space_id,const byte * flags,const byte * name,ulint len,const byte * new_name,ulint new_len)820 static void backup_file_op(ulint space_id, const byte* flags,
821 const byte* name, ulint len,
822 const byte* new_name, ulint new_len)
823 {
824
825 ut_ad(!flags || !new_name);
826 ut_ad(name);
827 ut_ad(len);
828 ut_ad(!new_name == !new_len);
829 pthread_mutex_lock(&backup_mutex);
830
831 if (flags) {
832 ddl_tracker.id_to_name[space_id] = filename_to_spacename(name, len);
833 msg("DDL tracking : create %zu \"%.*s\": %x",
834 space_id, int(len), name, mach_read_from_4(flags));
835 }
836 else if (new_name) {
837 ddl_tracker.id_to_name[space_id] = filename_to_spacename(new_name, new_len);
838 msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"",
839 space_id, int(len), name, int(new_len), new_name);
840 } else {
841 ddl_tracker.drops.insert(space_id);
842 msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name);
843 }
844 pthread_mutex_unlock(&backup_mutex);
845 }
846
847
848 /*
849 This callback is called if DDL operation is detected,
850 at the end of backup
851
852 Normally, DDL operations are blocked due to FTWRL,
853 but in rare cases of --no-lock, they are not.
854
855 We will abort backup in this case.
856 */
backup_file_op_fail(ulint space_id,const byte * flags,const byte * name,ulint len,const byte * new_name,ulint new_len)857 static void backup_file_op_fail(ulint space_id, const byte* flags,
858 const byte* name, ulint len,
859 const byte* new_name, ulint new_len)
860 {
861 bool fail;
862 if (flags) {
863 msg("DDL tracking : create %zu \"%.*s\": %x",
864 space_id, int(len), name, mach_read_from_4(flags));
865 std::string spacename = filename_to_spacename(name, len);
866 fail = !check_if_skip_table(spacename.c_str());
867 }
868 else if (new_name) {
869 msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"",
870 space_id, int(len), name, int(new_len), new_name);
871 std::string spacename = filename_to_spacename(name, len);
872 std::string new_spacename = filename_to_spacename(new_name, new_len);
873 fail = !check_if_skip_table(spacename.c_str()) || !check_if_skip_table(new_spacename.c_str());
874 }
875 else {
876 std::string spacename = filename_to_spacename(name, len);
877 fail = !check_if_skip_table(spacename.c_str());
878 msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name);
879 }
880 if (fail) {
881 ut_a(opt_no_lock);
882 die("DDL operation detected in the late phase of backup."
883 "Backup is inconsistent. Remove --no-lock option to fix.");
884 }
885 }
886
887
888 /** Callback whenever MLOG_INDEX_LOAD happens.
889 @param[in] space_id space id to check */
backup_optimized_ddl_op(ulint space_id)890 static void backup_optimized_ddl_op(ulint space_id)
891 {
892 pthread_mutex_lock(&backup_mutex);
893 ddl_tracker.optimized_ddl.insert(space_id);
894 pthread_mutex_unlock(&backup_mutex);
895 }
896
897 /*
898 Optimized DDL callback at the end of backup that
899 run with --no-lock. Usually aborts the backup.
900 */
backup_optimized_ddl_op_fail(ulint space_id)901 static void backup_optimized_ddl_op_fail(ulint space_id) {
902 msg("DDL tracking : optimized DDL on space %zu", space_id);
903 if (ddl_tracker.tables_in_backup.find(space_id) != ddl_tracker.tables_in_backup.end()) {
904 ut_a(opt_no_lock);
905 msg("ERROR : Optimized DDL operation detected in the late phase of backup."
906 "Backup is inconsistent. Remove --no-lock option to fix.");
907 exit(EXIT_FAILURE);
908 }
909 }
910
911
912 /*
913 Retrieve default data directory, to be used with --copy-back.
914
915 On Windows, default datadir is ..\data, relative to the
916 directory where mariabackup.exe is located(usually "bin")
917
918 Elsewhere, the compiled-in constant MYSQL_DATADIR is used.
919 */
get_default_datadir()920 static char *get_default_datadir() {
921 static char ddir[] = MYSQL_DATADIR;
922 #ifdef _WIN32
923 static char buf[MAX_PATH];
924 DWORD size = (DWORD)sizeof(buf) - 1;
925 if (GetModuleFileName(NULL, buf, size) <= size)
926 {
927 char *p;
928 if ((p = strrchr(buf, '\\')))
929 {
930 *p = 0;
931 if ((p = strrchr(buf, '\\')))
932 {
933 strncpy(p + 1, "data", buf + MAX_PATH - p);
934 return buf;
935 }
936 }
937 }
938 #endif
939 return ddir;
940 }
941
942
943 /* ======== Date copying thread context ======== */
944
945 typedef struct {
946 datafiles_iter_t *it;
947 uint num;
948 uint *count;
949 pthread_mutex_t* count_mutex;
950 os_thread_id_t id;
951 CorruptedPages *corrupted_pages;
952 } data_thread_ctxt_t;
953
954 /* ======== for option and variables ======== */
955 #include <../../client/client_priv.h>
956
957 enum options_xtrabackup
958 {
959 OPT_XTRA_TARGET_DIR= 1000, /* make sure it is larger
960 than OPT_MAX_CLIENT_OPTION */
961 OPT_XTRA_BACKUP,
962 OPT_XTRA_PREPARE,
963 OPT_XTRA_EXPORT,
964 OPT_XTRA_ROLLBACK_XA,
965 OPT_XTRA_PRINT_PARAM,
966 OPT_XTRA_USE_MEMORY,
967 OPT_XTRA_THROTTLE,
968 OPT_XTRA_LOG_COPY_INTERVAL,
969 OPT_XTRA_INCREMENTAL,
970 OPT_XTRA_INCREMENTAL_BASEDIR,
971 OPT_XTRA_EXTRA_LSNDIR,
972 OPT_XTRA_INCREMENTAL_DIR,
973 OPT_XTRA_TABLES,
974 OPT_XTRA_TABLES_FILE,
975 OPT_XTRA_DATABASES,
976 OPT_XTRA_DATABASES_FILE,
977 OPT_XTRA_PARALLEL,
978 OPT_XTRA_EXTENDED_VALIDATION,
979 OPT_XTRA_ENCRYPTED_BACKUP,
980 OPT_XTRA_STREAM,
981 OPT_XTRA_COMPRESS,
982 OPT_XTRA_COMPRESS_THREADS,
983 OPT_XTRA_COMPRESS_CHUNK_SIZE,
984 OPT_LOG,
985 OPT_INNODB,
986 OPT_INNODB_DATA_FILE_PATH,
987 OPT_INNODB_DATA_HOME_DIR,
988 OPT_INNODB_ADAPTIVE_HASH_INDEX,
989 OPT_INNODB_DOUBLEWRITE,
990 OPT_INNODB_FILE_PER_TABLE,
991 OPT_INNODB_FLUSH_METHOD,
992 OPT_INNODB_LOG_GROUP_HOME_DIR,
993 OPT_INNODB_MAX_DIRTY_PAGES_PCT,
994 OPT_INNODB_MAX_PURGE_LAG,
995 OPT_INNODB_STATUS_FILE,
996 OPT_INNODB_AUTOEXTEND_INCREMENT,
997 OPT_INNODB_BUFFER_POOL_SIZE,
998 OPT_INNODB_COMMIT_CONCURRENCY,
999 OPT_INNODB_CONCURRENCY_TICKETS,
1000 OPT_INNODB_FILE_IO_THREADS,
1001 OPT_INNODB_IO_CAPACITY,
1002 OPT_INNODB_READ_IO_THREADS,
1003 OPT_INNODB_WRITE_IO_THREADS,
1004 OPT_INNODB_USE_NATIVE_AIO,
1005 OPT_INNODB_PAGE_SIZE,
1006 OPT_INNODB_BUFFER_POOL_FILENAME,
1007 OPT_INNODB_LOCK_WAIT_TIMEOUT,
1008 OPT_INNODB_LOG_BUFFER_SIZE,
1009 OPT_INNODB_LOG_FILE_SIZE,
1010 OPT_INNODB_LOG_FILES_IN_GROUP,
1011 OPT_INNODB_OPEN_FILES,
1012 OPT_XTRA_DEBUG_SYNC,
1013 OPT_INNODB_CHECKSUM_ALGORITHM,
1014 OPT_INNODB_UNDO_DIRECTORY,
1015 OPT_INNODB_UNDO_TABLESPACES,
1016 OPT_INNODB_LOG_CHECKSUMS,
1017 OPT_XTRA_INCREMENTAL_FORCE_SCAN,
1018 OPT_DEFAULTS_GROUP,
1019 OPT_CLOSE_FILES,
1020 OPT_CORE_FILE,
1021
1022 OPT_COPY_BACK,
1023 OPT_MOVE_BACK,
1024 OPT_GALERA_INFO,
1025 OPT_SLAVE_INFO,
1026 OPT_NO_LOCK,
1027 OPT_SAFE_SLAVE_BACKUP,
1028 OPT_RSYNC,
1029 OPT_FORCE_NON_EMPTY_DIRS,
1030 OPT_NO_VERSION_CHECK,
1031 OPT_NO_BACKUP_LOCKS,
1032 OPT_DECOMPRESS,
1033 OPT_INCREMENTAL_HISTORY_NAME,
1034 OPT_INCREMENTAL_HISTORY_UUID,
1035 OPT_REMOVE_ORIGINAL,
1036 OPT_LOCK_WAIT_QUERY_TYPE,
1037 OPT_KILL_LONG_QUERY_TYPE,
1038 OPT_HISTORY,
1039 OPT_KILL_LONG_QUERIES_TIMEOUT,
1040 OPT_LOCK_WAIT_TIMEOUT,
1041 OPT_LOCK_WAIT_THRESHOLD,
1042 OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
1043 OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
1044 OPT_BINLOG_INFO,
1045 OPT_XB_SECURE_AUTH,
1046
1047 OPT_XTRA_TABLES_EXCLUDE,
1048 OPT_XTRA_DATABASES_EXCLUDE,
1049 OPT_PROTOCOL,
1050 OPT_INNODB_COMPRESSION_LEVEL,
1051 OPT_LOCK_DDL_PER_TABLE,
1052 OPT_ROCKSDB_DATADIR,
1053 OPT_BACKUP_ROCKSDB,
1054 OPT_XTRA_CHECK_PRIVILEGES,
1055 OPT_XTRA_MYSQLD_ARGS,
1056 OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
1057 OPT_INNODB_FORCE_RECOVERY
1058 };
1059
1060 struct my_option xb_client_options[]= {
1061 {"verbose", 'V', "display verbose output", (G_PTR *) &verbose,
1062 (G_PTR *) &verbose, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
1063 {"version", 'v', "print version information",
1064 (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL,
1065 NO_ARG, 0, 0, 0, 0, 0, 0},
1066 {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory",
1067 (G_PTR *) &xtrabackup_target_dir, (G_PTR *) &xtrabackup_target_dir, 0,
1068 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1069 {"backup", OPT_XTRA_BACKUP, "take backup to target-dir",
1070 (G_PTR *) &xtrabackup_backup, (G_PTR *) &xtrabackup_backup, 0, GET_BOOL,
1071 NO_ARG, 0, 0, 0, 0, 0, 0},
1072 {"prepare", OPT_XTRA_PREPARE,
1073 "prepare a backup for starting mysql server on the backup.",
1074 (G_PTR *) &xtrabackup_prepare, (G_PTR *) &xtrabackup_prepare, 0, GET_BOOL,
1075 NO_ARG, 0, 0, 0, 0, 0, 0},
1076 {"export", OPT_XTRA_EXPORT,
1077 "create files to import to another database when prepare.",
1078 (G_PTR *) &xtrabackup_export, (G_PTR *) &xtrabackup_export, 0, GET_BOOL,
1079 NO_ARG, 0, 0, 0, 0, 0, 0},
1080 {"rollback-xa", OPT_XTRA_ROLLBACK_XA,
1081 "Rollback prepared XA's on --prepare. "
1082 "After preparing target directory with this option "
1083 "it can no longer be a base for incremental backup.",
1084 (G_PTR *) &xtrabackup_rollback_xa, (G_PTR *) &xtrabackup_rollback_xa, 0,
1085 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1086 {"print-param", OPT_XTRA_PRINT_PARAM,
1087 "print parameter of mysqld needed for copyback.",
1088 (G_PTR *) &xtrabackup_print_param, (G_PTR *) &xtrabackup_print_param, 0,
1089 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1090 {"use-memory", OPT_XTRA_USE_MEMORY,
1091 "The value is used instead of buffer_pool_size",
1092 (G_PTR *) &xtrabackup_use_memory, (G_PTR *) &xtrabackup_use_memory, 0,
1093 GET_LL, REQUIRED_ARG, 100 * 1024 * 1024L, 1024 * 1024L, LONGLONG_MAX, 0,
1094 1024 * 1024L, 0},
1095 {"throttle", OPT_XTRA_THROTTLE,
1096 "limit count of IO operations (pairs of read&write) per second to IOS "
1097 "values (for '--backup')",
1098 (G_PTR *) &xtrabackup_throttle, (G_PTR *) &xtrabackup_throttle, 0,
1099 GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0},
1100 {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
1101 (G_PTR *) &log_ignored_opt, (G_PTR *) &log_ignored_opt, 0, GET_STR,
1102 OPT_ARG, 0, 0, 0, 0, 0, 0},
1103 {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL,
1104 "time interval between checks done by log copying thread in milliseconds "
1105 "(default is 1 second).",
1106 (G_PTR *) &xtrabackup_log_copy_interval,
1107 (G_PTR *) &xtrabackup_log_copy_interval, 0, GET_LONG, REQUIRED_ARG, 1000,
1108 0, LONG_MAX, 0, 1, 0},
1109 {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR,
1110 "(for --backup): save an extra copy of the xtrabackup_checkpoints file "
1111 "in this directory.",
1112 (G_PTR *) &xtrabackup_extra_lsndir, (G_PTR *) &xtrabackup_extra_lsndir, 0,
1113 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1114 {"incremental-lsn", OPT_XTRA_INCREMENTAL,
1115 "(for --backup): copy only .ibd pages newer than specified LSN "
1116 "'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is "
1117 "impossible to diagnose this, causing the backup to be unusable. Be "
1118 "careful!",
1119 (G_PTR *) &xtrabackup_incremental, (G_PTR *) &xtrabackup_incremental, 0,
1120 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1121 {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR,
1122 "(for --backup): copy only .ibd pages newer than backup at specified "
1123 "directory.",
1124 (G_PTR *) &xtrabackup_incremental_basedir,
1125 (G_PTR *) &xtrabackup_incremental_basedir, 0, GET_STR, REQUIRED_ARG, 0, 0,
1126 0, 0, 0, 0},
1127 {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR,
1128 "(for --prepare): apply .delta files and logfile in the specified "
1129 "directory.",
1130 (G_PTR *) &xtrabackup_incremental_dir,
1131 (G_PTR *) &xtrabackup_incremental_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
1132 0, 0, 0},
1133 {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.",
1134 (G_PTR *) &xtrabackup_tables, (G_PTR *) &xtrabackup_tables, 0, GET_STR,
1135 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1136 {"tables_file", OPT_XTRA_TABLES_FILE,
1137 "filtering by list of the exact database.table name in the file.",
1138 (G_PTR *) &xtrabackup_tables_file, (G_PTR *) &xtrabackup_tables_file, 0,
1139 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1140 {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.",
1141 (G_PTR *) &xtrabackup_databases, (G_PTR *) &xtrabackup_databases, 0,
1142 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1143 {"databases_file", OPT_XTRA_DATABASES_FILE,
1144 "filtering by list of databases in the file.",
1145 (G_PTR *) &xtrabackup_databases_file,
1146 (G_PTR *) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
1147 0, 0, 0},
1148 {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE,
1149 "filtering by regexp for table names. "
1150 "Operates the same way as --tables, but matched names are excluded from "
1151 "backup. "
1152 "Note that this option has a higher priority than --tables.",
1153 (G_PTR *) &xtrabackup_tables_exclude,
1154 (G_PTR *) &xtrabackup_tables_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
1155 0, 0, 0},
1156 {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE,
1157 "Excluding databases based on name, "
1158 "Operates the same way as --databases, but matched names are excluded "
1159 "from backup. "
1160 "Note that this option has a higher priority than --databases.",
1161 (G_PTR *) &xtrabackup_databases_exclude,
1162 (G_PTR *) &xtrabackup_databases_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0,
1163 0, 0, 0, 0},
1164
1165 {"stream", OPT_XTRA_STREAM,
1166 "Stream all backup files to the standard output "
1167 "in the specified format."
1168 "Supported format is 'mbstream' or 'xbstream'.",
1169 (G_PTR *) &xtrabackup_stream_str, (G_PTR *) &xtrabackup_stream_str, 0,
1170 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1171
1172 {"compress", OPT_XTRA_COMPRESS,
1173 "Compress individual backup files using the "
1174 "specified compression algorithm. Currently the only supported algorithm "
1175 "is 'quicklz'. It is also the default algorithm, i.e. the one used when "
1176 "--compress is used without an argument.",
1177 (G_PTR *) &xtrabackup_compress_alg, (G_PTR *) &xtrabackup_compress_alg, 0,
1178 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1179
1180 {"compress-threads", OPT_XTRA_COMPRESS_THREADS,
1181 "Number of threads for parallel data compression. The default value is "
1182 "1.",
1183 (G_PTR *) &xtrabackup_compress_threads,
1184 (G_PTR *) &xtrabackup_compress_threads, 0, GET_UINT, REQUIRED_ARG, 1, 1,
1185 UINT_MAX, 0, 0, 0},
1186
1187 {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE,
1188 "Size of working buffer(s) for compression threads in bytes. The default "
1189 "value is 64K.",
1190 (G_PTR *) &xtrabackup_compress_chunk_size,
1191 (G_PTR *) &xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG,
1192 (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
1193
1194 {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN,
1195 "Perform a full-scan incremental backup even in the presence of changed "
1196 "page bitmap data",
1197 (G_PTR *) &xtrabackup_incremental_force_scan,
1198 (G_PTR *) &xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, 0, 0,
1199 0, 0, 0, 0},
1200
1201 {"close_files", OPT_CLOSE_FILES,
1202 "do not keep files opened. Use at your own "
1203 "risk.",
1204 (G_PTR *) &xb_close_files, (G_PTR *) &xb_close_files, 0, GET_BOOL, NO_ARG,
1205 0, 0, 0, 0, 0, 0},
1206
1207 {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0,
1208 GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
1209
1210 {"copy-back", OPT_COPY_BACK,
1211 "Copy all the files in a previously made "
1212 "backup from the backup directory to their original locations.",
1213 (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0,
1214 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1215
1216 {"move-back", OPT_MOVE_BACK,
1217 "Move all the files in a previously made "
1218 "backup from the backup directory to the actual datadir location. "
1219 "Use with caution, as it removes backup files.",
1220 (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0,
1221 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1222
1223 {"galera-info", OPT_GALERA_INFO,
1224 "This options creates the "
1225 "xtrabackup_galera_info file which contains the local node state at "
1226 "the time of the backup. Option should be used when performing the "
1227 "backup of MariaDB Galera Cluster. Has no effect when backup locks "
1228 "are used to create the backup.",
1229 (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0, GET_BOOL,
1230 NO_ARG, 0, 0, 0, 0, 0, 0},
1231
1232 {"slave-info", OPT_SLAVE_INFO,
1233 "This option is useful when backing "
1234 "up a replication slave server. It prints the binary log position "
1235 "and name of the master server. It also writes this information to "
1236 "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. "
1237 "A new slave for this master can be set up by starting a slave server "
1238 "on this backup and issuing a \"CHANGE MASTER\" command with the "
1239 "binary log position saved in the \"xtrabackup_slave_info\" file.",
1240 (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0, GET_BOOL, NO_ARG,
1241 0, 0, 0, 0, 0, 0},
1242
1243 {"no-lock", OPT_NO_LOCK,
1244 "Use this option to disable table lock "
1245 "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
1246 "tables are InnoDB and you DO NOT CARE about the binary log "
1247 "position of the backup. This option shouldn't be used if there "
1248 "are any DDL statements being executed or if any updates are "
1249 "happening on non-InnoDB tables (this includes the system MyISAM "
1250 "tables in the mysql database), otherwise it could lead to an "
1251 "inconsistent backup. If you are considering to use --no-lock "
1252 "because your backups are failing to acquire the lock, this could "
1253 "be because of incoming replication events preventing the lock "
1254 "from succeeding. Please try using --safe-slave-backup to "
1255 "momentarily stop the replication slave thread, this may help "
1256 "the backup to succeed and you then don't need to resort to "
1257 "using this option.",
1258 (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0, GET_BOOL, NO_ARG, 0, 0,
1259 0, 0, 0, 0},
1260
1261 {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP,
1262 "Stop slave SQL thread "
1263 "and wait to start backup until Slave_open_temp_tables in "
1264 "\"SHOW STATUS\" is zero. If there are no open temporary tables, "
1265 "the backup will take place, otherwise the SQL thread will be "
1266 "started and stopped until there are no open temporary tables. "
1267 "The backup will fail if Slave_open_temp_tables does not become "
1268 "zero after --safe-slave-backup-timeout seconds. The slave SQL "
1269 "thread will be restarted when the backup finishes.",
1270 (uchar *) &opt_safe_slave_backup, (uchar *) &opt_safe_slave_backup, 0,
1271 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1272
1273 {"rsync", OPT_RSYNC,
1274 "Uses the rsync utility to optimize local file "
1275 "transfers. When this option is specified, innobackupex uses rsync "
1276 "to copy all non-InnoDB files instead of spawning a separate cp for "
1277 "each file, which can be much faster for servers with a large number "
1278 "of databases or tables. This option cannot be used together with "
1279 "--stream.",
1280 (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0,
1281 0, 0, 0},
1282
1283 {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS,
1284 "This "
1285 "option, when specified, makes --copy-back or --move-back transfer "
1286 "files to non-empty directories. Note that no existing files will be "
1287 "overwritten. If --copy-back or --move-back has to copy a file from "
1288 "the backup directory which already exists in the destination "
1289 "directory, it will still fail with an error.",
1290 (uchar *) &opt_force_non_empty_dirs, (uchar *) &opt_force_non_empty_dirs,
1291 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1292
1293 {"no-version-check", OPT_NO_VERSION_CHECK,
1294 "This option disables the "
1295 "version check which is enabled by the --version-check option.",
1296 (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL,
1297 NO_ARG, 0, 0, 0, 0, 0, 0},
1298
1299 {"no-backup-locks", OPT_NO_BACKUP_LOCKS,
1300 "This option controls if "
1301 "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
1302 "on the backup stage. The option has no effect when backup locks are "
1303 "not supported by the server. This option is enabled by default, "
1304 "disable with --no-backup-locks.",
1305 (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0,
1306 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1307
1308 {"decompress", OPT_DECOMPRESS,
1309 "Decompresses all files with the .qp "
1310 "extension in a backup previously made with the --compress option.",
1311 (uchar *) &opt_decompress, (uchar *) &opt_decompress, 0, GET_BOOL, NO_ARG,
1312 0, 0, 0, 0, 0, 0},
1313
1314 {"user", 'u',
1315 "This option specifies the MySQL username used "
1316 "when connecting to the server, if that's not the current user. "
1317 "The option accepts a string argument. See mysql --help for details.",
1318 (uchar *) &opt_user, (uchar *) &opt_user, 0, GET_STR, REQUIRED_ARG, 0, 0,
1319 0, 0, 0, 0},
1320
1321 {"host", 'H',
1322 "This option specifies the host to use when "
1323 "connecting to the database server with TCP/IP. The option accepts "
1324 "a string argument. See mysql --help for details.",
1325 (uchar *) &opt_host, (uchar *) &opt_host, 0, GET_STR, REQUIRED_ARG, 0, 0,
1326 0, 0, 0, 0},
1327
1328 {"port", 'P',
1329 "This option specifies the port to use when "
1330 "connecting to the database server with TCP/IP. The option accepts "
1331 "a string argument. See mysql --help for details.",
1332 &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1333
1334 {"password", 'p',
1335 "This option specifies the password to use "
1336 "when connecting to the database. It accepts a string argument. "
1337 "See mysql --help for details.",
1338 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1339
1340 {"protocol", OPT_PROTOCOL,
1341 "The protocol to use for connection (tcp, socket, pipe, memory).", 0, 0,
1342 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1343
1344 {"socket", 'S',
1345 "This option specifies the socket to use when "
1346 "connecting to the local database server with a UNIX domain socket. "
1347 "The option accepts a string argument. See mysql --help for details.",
1348 (uchar *) &opt_socket, (uchar *) &opt_socket, 0, GET_STR, REQUIRED_ARG, 0,
1349 0, 0, 0, 0, 0},
1350
1351 {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME,
1352 "This option specifies the name of the backup series stored in the "
1353 "PERCONA_SCHEMA.xtrabackup_history history record to base an "
1354 "incremental backup on. Xtrabackup will search the history table "
1355 "looking for the most recent (highest innodb_to_lsn), successful "
1356 "backup in the series and take the to_lsn value to use as the "
1357 "starting lsn for the incremental backup. This will be mutually "
1358 "exclusive with --incremental-history-uuid, --incremental-basedir "
1359 "and --incremental-lsn. If no valid lsn can be found (no series by "
1360 "that name, no successful backups by that name), an error will be returned."
1361 " It is used with the --incremental option.",
1362 (uchar *) &opt_incremental_history_name,
1363 (uchar *) &opt_incremental_history_name, 0, GET_STR, REQUIRED_ARG, 0, 0,
1364 0, 0, 0, 0},
1365
1366 {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID,
1367 "This option specifies the UUID of the specific history record "
1368 "stored in the PERCONA_SCHEMA.xtrabackup_history to base an "
1369 "incremental backup on. --incremental-history-name, "
1370 "--incremental-basedir and --incremental-lsn. If no valid lsn can be "
1371 "found (no success record with that uuid), an error will be returned."
1372 " It is used with the --incremental option.",
1373 (uchar *) &opt_incremental_history_uuid,
1374 (uchar *) &opt_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0,
1375 0, 0, 0, 0},
1376
1377 {"remove-original", OPT_REMOVE_ORIGINAL,
1378 "Remove .qp files after decompression.", (uchar *) &opt_remove_original,
1379 (uchar *) &opt_remove_original, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1380
1381 {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
1382 "This option specifies which types of queries are allowed to complete "
1383 "before innobackupex will issue the global lock. Default is all.",
1384 (uchar *) &opt_lock_wait_query_type, (uchar *) &opt_lock_wait_query_type,
1385 &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0,
1386 0},
1387
1388 {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
1389 "This option specifies which types of queries should be killed to "
1390 "unblock the global lock. Default is \"all\".",
1391 (uchar *) &opt_kill_long_query_type, (uchar *) &opt_kill_long_query_type,
1392 &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0,
1393 0, 0},
1394
1395 {"history", OPT_HISTORY,
1396 "This option enables the tracking of backup history in the "
1397 "PERCONA_SCHEMA.xtrabackup_history table. An optional history "
1398 "series name may be specified that will be placed with the history "
1399 "record for the current backup being taken.",
1400 NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1401
1402 {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
1403 "This option specifies the number of seconds innobackupex waits "
1404 "between starting FLUSH TABLES WITH READ LOCK and killing those "
1405 "queries that block it. Default is 0 seconds, which means "
1406 "innobackupex will not attempt to kill any queries.",
1407 (uchar *) &opt_kill_long_queries_timeout,
1408 (uchar *) &opt_kill_long_queries_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0,
1409 0, 0, 0, 0},
1410
1411 {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
1412 "This option specifies time in seconds that innobackupex should wait "
1413 "for queries that would block FTWRL before running it. If there are "
1414 "still such queries when the timeout expires, innobackupex terminates "
1415 "with an error. Default is 0, in which case innobackupex does not "
1416 "wait for queries to complete and starts FTWRL immediately.",
1417 (uchar *) &opt_lock_wait_timeout, (uchar *) &opt_lock_wait_timeout, 0,
1418 GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1419
1420 {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
1421 "This option specifies the query run time threshold which is used by "
1422 "innobackupex to detect long-running queries with a non-zero value "
1423 "of --ftwrl-wait-timeout. FTWRL is not started until such "
1424 "long-running queries exist. This option has no effect if "
1425 "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
1426 (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0,
1427 GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0},
1428
1429
1430 {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
1431 "How many seconds --safe-slave-backup should wait for "
1432 "Slave_open_temp_tables to become zero. (default 300)",
1433 (uchar *) &opt_safe_slave_backup_timeout,
1434 (uchar *) &opt_safe_slave_backup_timeout, 0, GET_UINT, REQUIRED_ARG, 300,
1435 0, 0, 0, 0, 0},
1436
1437 {"binlog-info", OPT_BINLOG_INFO,
1438 "This option controls how backup should retrieve server's binary log "
1439 "coordinates corresponding to the backup. Possible values are OFF, ON, "
1440 "LOCKLESS and AUTO.",
1441 &opt_binlog_info, &opt_binlog_info, &binlog_info_typelib, GET_ENUM,
1442 OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0},
1443
1444 {"secure-auth", OPT_XB_SECURE_AUTH,
1445 "Refuse client connecting to server if it"
1446 " uses old (pre-4.1.1) protocol.",
1447 &opt_secure_auth, &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
1448 0},
1449
1450 {"log-innodb-page-corruption", OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
1451 "Continue backup if innodb corrupted pages are found. The pages are "
1452 "logged in " MB_CORRUPTED_PAGES_FILE
1453 " and backup is finished with error. "
1454 "--prepare will try to fix corrupted pages. If " MB_CORRUPTED_PAGES_FILE
1455 " exists after --prepare in base backup directory, backup still contains "
1456 "corrupted pages and can not be considered as consistent.",
1457 &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0,
1458 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1459
1460 #define MYSQL_CLIENT
1461 #include "sslopt-longopts.h"
1462 #undef MYSQL_CLIENT
1463 {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
1464
1465 uint xb_client_options_count = array_elements(xb_client_options);
1466
1467 #ifndef DBUG_OFF
1468 /** Parameters to DBUG */
1469 static const char *dbug_option;
1470 #endif
1471
1472 struct my_option xb_server_options[] =
1473 {
1474 {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home,
1475 (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1476 {"tmpdir", 't',
1477 "Path for temporary files. Several paths may be specified, separated by a "
1478 #if defined(__WIN__) || defined(OS2) || defined(__NETWARE__)
1479 "semicolon (;)"
1480 #else
1481 "colon (:)"
1482 #endif
1483 ", in this case they are used in a round-robin fashion.",
1484 (G_PTR*) &opt_mysql_tmpdir,
1485 (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1486 {"parallel", OPT_XTRA_PARALLEL,
1487 "Number of threads to use for parallel datafiles transfer. "
1488 "The default value is 1.",
1489 (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
1490 REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
1491
1492 {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION,
1493 "Enable extended validation for Innodb data pages during backup phase. "
1494 "Will slow down backup considerably, in case encryption is used. "
1495 "May fail if tables are created during the backup.",
1496 (G_PTR*)&opt_extended_validation,
1497 (G_PTR*)&opt_extended_validation,
1498 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
1499
1500 {"encrypted_backup", OPT_XTRA_ENCRYPTED_BACKUP,
1501 "In --backup, assume that nonzero key_version implies that the page"
1502 " is encrypted. Use --backup --skip-encrypted-backup to allow"
1503 " copying unencrypted that were originally created before MySQL 5.1.48.",
1504 (G_PTR*)&opt_encrypted_backup,
1505 (G_PTR*)&opt_encrypted_backup,
1506 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0},
1507
1508 {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
1509 (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
1510 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1511
1512 {"log_bin", OPT_LOG, "Base name for the log sequence",
1513 &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1514
1515 {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility",
1516 (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0,
1517 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1518 #ifdef BTR_CUR_HASH_ADAPT
1519 {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX,
1520 "Enable InnoDB adaptive hash index (enabled by default). "
1521 "Disable with --skip-innodb-adaptive-hash-index.",
1522 &btr_search_enabled,
1523 &btr_search_enabled,
1524 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
1525 #endif /* BTR_CUR_HASH_ADAPT */
1526 {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT,
1527 "Data file autoextend increment in megabytes",
1528 (G_PTR*) &sys_tablespace_auto_extend_increment,
1529 (G_PTR*) &sys_tablespace_auto_extend_increment,
1530 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0},
1531 {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH,
1532 "Path to individual files and their sizes.", &innobase_data_file_path,
1533 &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1534 {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR,
1535 "The common part for InnoDB table spaces.", &innobase_data_home_dir,
1536 &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1537 {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE,
1538 "Enable InnoDB doublewrite buffer during --prepare.",
1539 (G_PTR*) &innobase_use_doublewrite,
1540 (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1541 {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY,
1542 "Number of IOPs the server can do. Tunes the background IO rate",
1543 (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity,
1544 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0},
1545 {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS,
1546 "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads,
1547 (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0,
1548 1, 0},
1549 {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS,
1550 "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads,
1551 (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
1552 1, 0},
1553 {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS,
1554 "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads,
1555 (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
1556 1, 0},
1557 {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE,
1558 "Stores each InnoDB table to an .ibd file in the database dir.",
1559 (G_PTR*) &innobase_file_per_table,
1560 (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG,
1561 FALSE, 0, 0, 0, 0, 0},
1562
1563 {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD,
1564 "With which method to flush data.",
1565 &srv_file_flush_method, &srv_file_flush_method,
1566 &innodb_flush_method_typelib, GET_ENUM, REQUIRED_ARG,
1567 IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_FSYNC), 0, 0, 0, 0, 0},
1568
1569 {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE,
1570 "The size of the buffer which InnoDB uses to write log to the log files on disk.",
1571 (G_PTR*) &srv_log_buffer_size, (G_PTR*) &srv_log_buffer_size, 0,
1572 GET_ULONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0},
1573 {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
1574 "Ignored for mysqld option compatibility",
1575 (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
1576 GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, log_group_max_size, 0,
1577 UNIV_PAGE_SIZE_MAX, 0},
1578 {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP,
1579 "Ignored for mysqld option compatibility",
1580 &srv_n_log_files, &srv_n_log_files,
1581 0, GET_LONG, REQUIRED_ARG, 1, 1, 100, 0, 1, 0},
1582 {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
1583 "Path to InnoDB log files.", &srv_log_group_home_dir,
1584 &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1585 {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT,
1586 "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct,
1587 (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0},
1588 {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO,
1589 "Use native AIO if supported on this platform.",
1590 (G_PTR*) &srv_use_native_aio,
1591 (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG,
1592 TRUE, 0, 0, 0, 0, 0},
1593 {"innodb_page_size", OPT_INNODB_PAGE_SIZE,
1594 "The universal page size of the database.",
1595 (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0,
1596 /* Use GET_LL to support numeric suffixes in 5.6 */
1597 GET_LL, REQUIRED_ARG,
1598 (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0},
1599 {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME,
1600 "Ignored for mysqld option compatibility",
1601 (G_PTR*) &innobase_buffer_pool_filename,
1602 (G_PTR*) &innobase_buffer_pool_filename,
1603 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1604
1605 #ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */
1606 {"dbug", '#', "Built in DBUG debugger.",
1607 &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG,
1608 0, 0, 0, 0, 0, 0},
1609 #endif
1610
1611 {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM,
1612 "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
1613 "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
1614 &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
1615 REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},
1616
1617 {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
1618 "Directory where undo tablespace files live, this path can be absolute.",
1619 &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0,
1620 0},
1621
1622 {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES,
1623 "Number of undo tablespaces to use.",
1624 (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces,
1625 0, GET_ULONG, REQUIRED_ARG, 0, 0, 126, 0, 1, 0},
1626
1627 {"innodb_compression_level", OPT_INNODB_COMPRESSION_LEVEL,
1628 "Compression level used for zlib compression.",
1629 (G_PTR*)&page_zip_level, (G_PTR*)&page_zip_level,
1630 0, GET_UINT, REQUIRED_ARG, 6, 0, 9, 0, 0, 0},
1631
1632 {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").",
1633 (G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
1634 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1635
1636 {"plugin-dir", OPT_PLUGIN_DIR,
1637 "Server plugin directory. Used to load encryption plugin during 'prepare' phase."
1638 "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)",
1639 &xb_plugin_dir, &xb_plugin_dir,
1640 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1641
1642 {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS,
1643 "Whether to require checksums for InnoDB redo log blocks",
1644 &innodb_log_checksums, &innodb_log_checksums,
1645 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 },
1646
1647 {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
1648 "descriptors to reserve with setrlimit().",
1649 (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
1650 REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},
1651
1652 {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table "
1653 "before backup starts to copy it and until the backup is completed.",
1654 (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
1655 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1656
1657 {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
1658 "This option is only used with --copy-back or --move-back option",
1659 &xb_rocksdb_datadir, &xb_rocksdb_datadir,
1660 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1661
1662 { "rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
1663 "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
1664 &xb_backup_rocksdb, &xb_backup_rocksdb,
1665 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
1666
1667 {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user "
1668 "privileges fro the backup user",
1669 &opt_check_privileges, &opt_check_privileges,
1670 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
1671
1672 {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY,
1673 "(for --prepare): Crash recovery mode (ignores "
1674 "page corruption; for emergencies only).",
1675 (G_PTR*)&srv_force_recovery,
1676 (G_PTR*)&srv_force_recovery,
1677 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
1678
1679 {"mysqld-args", OPT_XTRA_MYSQLD_ARGS,
1680 "All arguments that follow this argument are considered as server "
1681 "options, and if some of them are not supported by mariabackup, they "
1682 "will be ignored.",
1683 (G_PTR *) &xtrabackup_mysqld_args, (G_PTR *) &xtrabackup_mysqld_args, 0,
1684 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1685
1686 {"help", '?',
1687 "Display this help and exit.",
1688 (G_PTR *) &xtrabackup_help, (G_PTR *) &xtrabackup_help, 0,
1689 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1690
1691 { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
1692 };
1693
1694 uint xb_server_options_count = array_elements(xb_server_options);
1695
1696
1697 static std::set<std::string> tables_for_export;
1698
append_export_table(const char * dbname,const char * tablename,bool is_remote,bool set_size)1699 static void append_export_table(const char *dbname, const char *tablename,
1700 bool is_remote, bool set_size)
1701 {
1702 if(dbname && tablename && !is_remote)
1703 {
1704 char buf[3*FN_REFLEN];
1705 snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename);
1706 // trim .ibd
1707 char *p=strrchr(buf, '.');
1708 if (p) *p=0;
1709
1710 std::string name=ut_get_name(0, buf);
1711 /* Strip partition name comment from table name, if any */
1712 if (ends_with(name.c_str(), "*/"))
1713 {
1714 size_t pos= name.rfind("/*");
1715 if (pos != std::string::npos)
1716 name.resize(pos);
1717 }
1718 tables_for_export.insert(name);
1719 }
1720 }
1721
1722
1723 #define BOOTSTRAP_FILENAME "mariabackup_prepare_for_export.sql"
1724
create_bootstrap_file()1725 static int create_bootstrap_file()
1726 {
1727 FILE *f= fopen(BOOTSTRAP_FILENAME,"wb");
1728 if(!f)
1729 return -1;
1730
1731 fputs("SET NAMES UTF8;\n",f);
1732 enumerate_ibd_files(append_export_table);
1733 for (std::set<std::string>::iterator it = tables_for_export.begin();
1734 it != tables_for_export.end(); it++)
1735 {
1736 const char *tab = it->c_str();
1737 fprintf(f,
1738 "BEGIN NOT ATOMIC "
1739 "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;"
1740 "FLUSH TABLES %s FOR EXPORT;"
1741 "END;\n"
1742 "UNLOCK TABLES;\n",
1743 tab);
1744 }
1745 fclose(f);
1746 return 0;
1747 }
1748
prepare_export()1749 static int prepare_export()
1750 {
1751 int err= -1;
1752
1753 char cmdline[2*FN_REFLEN];
1754 FILE *outf;
1755
1756 if (create_bootstrap_file())
1757 return -1;
1758
1759 // Process defaults-file , it can have some --lc-language stuff,
1760 // which is* unfortunately* still necessary to get mysqld up
1761 if (strncmp(orig_argv1,"--defaults-file=", 16) == 0)
1762 {
1763 snprintf(cmdline, sizeof cmdline,
1764 IF_WIN("\"","") "\"%s\" --mysqld \"%s\""
1765 " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1766 " --innodb --innodb-fast-shutdown=0 --loose-partition"
1767 " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1768 " --console --skip-log-error --skip-log-bin --bootstrap %s< "
1769 BOOTSTRAP_FILENAME IF_WIN("\"",""),
1770 mariabackup_exe,
1771 orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
1772 xtrabackup_use_memory,
1773 (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
1774 }
1775 else
1776 {
1777 snprintf(cmdline, sizeof cmdline,
1778 IF_WIN("\"","") "\"%s\" --mysqld"
1779 " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1780 " --innodb --innodb-fast-shutdown=0 --loose-partition"
1781 " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1782 " --console --log-error= --skip-log-bin --bootstrap %s< "
1783 BOOTSTRAP_FILENAME IF_WIN("\"",""),
1784 mariabackup_exe,
1785 (my_defaults_group_suffix?my_defaults_group_suffix:""),
1786 xtrabackup_use_memory,
1787 (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
1788 }
1789
1790 msg("Prepare export : executing %s\n", cmdline);
1791 fflush(stderr);
1792
1793 outf= popen(cmdline,"r");
1794 if (!outf)
1795 goto end;
1796
1797 char outline[FN_REFLEN];
1798 while(fgets(outline, sizeof(outline)-1, outf))
1799 fprintf(stderr,"%s",outline);
1800
1801 err = pclose(outf);
1802 end:
1803 unlink(BOOTSTRAP_FILENAME);
1804 return err;
1805 }
1806
1807 static const char *xb_client_default_groups[]= {
1808 "client", "client-server", "client-mariadb", 0, 0, 0};
1809
1810 static const char *backup_default_groups[]= {
1811 "xtrabackup", "mariabackup", "mariadb-backup", 0, 0, 0};
1812
print_version(void)1813 static void print_version(void)
1814 {
1815 fprintf(stderr, "%s based on MariaDB server %s %s (%s)\n",
1816 my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
1817 }
1818
usage(void)1819 static void usage(void)
1820 {
1821 puts("Open source backup tool for InnoDB and XtraDB\n\
1822 \n\
1823 Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\
1824 Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\
1825 \n\
1826 This program is free software; you can redistribute it and/or\n\
1827 modify it under the terms of the GNU General Public License\n\
1828 as published by the Free Software Foundation version 2\n\
1829 of the License.\n\
1830 \n\
1831 This program is distributed in the hope that it will be useful,\n\
1832 but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
1833 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
1834 GNU General Public License for more details.\n\
1835 \n\
1836 You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n");
1837
1838 printf("Usage: %s [--defaults-file=#] [--backup | --prepare | --copy-back | --move-back] [OPTIONS]\n",my_progname);
1839 print_defaults("my", load_default_groups);
1840 my_print_help(xb_client_options);
1841 my_print_help(xb_server_options);
1842 my_print_variables(xb_server_options);
1843 my_print_variables(xb_client_options);
1844 }
1845
1846 #define ADD_PRINT_PARAM_OPT(value) \
1847 { \
1848 print_param_str << opt->name << "=" << value << "\n"; \
1849 param_set.insert(opt->name); \
1850 }
1851
1852 /************************************************************************
1853 Check if parameter is set in defaults file or via command line argument
1854 @return true if parameter is set. */
1855 bool
check_if_param_set(const char * param)1856 check_if_param_set(const char *param)
1857 {
1858 return param_set.find(param) != param_set.end();
1859 }
1860
1861 my_bool
xb_get_one_option(int optid,const struct my_option * opt,char * argument)1862 xb_get_one_option(int optid,
1863 const struct my_option *opt __attribute__((unused)),
1864 char *argument)
1865 {
1866 switch(optid) {
1867 case 'h':
1868 strmake(mysql_real_data_home,argument, FN_REFLEN - 1);
1869 mysql_data_home= mysql_real_data_home;
1870
1871 ADD_PRINT_PARAM_OPT(mysql_real_data_home);
1872 break;
1873
1874 case 't':
1875
1876 ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir);
1877 break;
1878
1879 case OPT_INNODB_DATA_HOME_DIR:
1880
1881 ADD_PRINT_PARAM_OPT(innobase_data_home_dir);
1882 break;
1883
1884 case OPT_INNODB_DATA_FILE_PATH:
1885
1886 ADD_PRINT_PARAM_OPT(innobase_data_file_path);
1887 break;
1888
1889 case OPT_INNODB_LOG_GROUP_HOME_DIR:
1890
1891 ADD_PRINT_PARAM_OPT(srv_log_group_home_dir);
1892 break;
1893
1894 case OPT_INNODB_LOG_FILES_IN_GROUP:
1895 case OPT_INNODB_LOG_FILE_SIZE:
1896 break;
1897
1898 case OPT_INNODB_FLUSH_METHOD:
1899 ut_a(srv_file_flush_method
1900 <= IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT_NO_FSYNC));
1901 ADD_PRINT_PARAM_OPT(innodb_flush_method_names[srv_file_flush_method]);
1902 break;
1903
1904 case OPT_INNODB_PAGE_SIZE:
1905
1906 ADD_PRINT_PARAM_OPT(innobase_page_size);
1907 break;
1908
1909 case OPT_INNODB_UNDO_DIRECTORY:
1910
1911 ADD_PRINT_PARAM_OPT(srv_undo_dir);
1912 break;
1913
1914 case OPT_INNODB_UNDO_TABLESPACES:
1915
1916 ADD_PRINT_PARAM_OPT(srv_undo_tablespaces);
1917 break;
1918
1919 case OPT_INNODB_CHECKSUM_ALGORITHM:
1920
1921 ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32);
1922
1923 ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]);
1924 break;
1925
1926 case OPT_INNODB_COMPRESSION_LEVEL:
1927 ADD_PRINT_PARAM_OPT(page_zip_level);
1928 break;
1929
1930 case OPT_INNODB_BUFFER_POOL_FILENAME:
1931
1932 ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
1933 break;
1934
1935 case OPT_INNODB_FORCE_RECOVERY:
1936
1937 if (srv_force_recovery) {
1938 ADD_PRINT_PARAM_OPT(srv_force_recovery);
1939 }
1940 break;
1941
1942 case OPT_XTRA_TARGET_DIR:
1943 strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
1944 xtrabackup_target_dir= xtrabackup_real_target_dir;
1945 break;
1946 case OPT_XTRA_STREAM:
1947 if (!strcasecmp(argument, "mbstream") ||
1948 !strcasecmp(argument, "xbstream"))
1949 xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
1950 else
1951 {
1952 msg("Invalid --stream argument: %s", argument);
1953 return 1;
1954 }
1955 xtrabackup_stream = TRUE;
1956 break;
1957 case OPT_XTRA_COMPRESS:
1958 if (argument == NULL)
1959 xtrabackup_compress_alg = "quicklz";
1960 else if (strcasecmp(argument, "quicklz"))
1961 {
1962 msg("Invalid --compress argument: %s", argument);
1963 return 1;
1964 }
1965 xtrabackup_compress = TRUE;
1966 break;
1967 case OPT_DECOMPRESS:
1968 opt_decompress = TRUE;
1969 xtrabackup_decrypt_decompress = true;
1970 break;
1971 case (int) OPT_CORE_FILE:
1972 test_flags |= TEST_CORE_ON_SIGNAL;
1973 break;
1974 case OPT_HISTORY:
1975 if (argument) {
1976 opt_history = argument;
1977 } else {
1978 opt_history = "";
1979 }
1980 break;
1981 case 'p':
1982 opt_password = argument;
1983 break;
1984 case OPT_PROTOCOL:
1985 if (argument)
1986 {
1987 if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib,
1988 opt->name)) <= 0)
1989 {
1990 sf_leaking_memory= 1; /* no memory leak reports here */
1991 exit(1);
1992 }
1993 }
1994 break;
1995 #define MYSQL_CLIENT
1996 #include "sslopt-case.h"
1997 #undef MYSQL_CLIENT
1998
1999 case '?':
2000 usage();
2001 exit(EXIT_SUCCESS);
2002 break;
2003 case 'v':
2004 print_version();
2005 exit(EXIT_SUCCESS);
2006 break;
2007 default:
2008 break;
2009 }
2010 return 0;
2011 }
2012
innodb_init_param()2013 static bool innodb_init_param()
2014 {
2015 srv_is_being_started = TRUE;
2016 /* === some variables from mysqld === */
2017 memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));
2018
2019 if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) {
2020 msg("init_tmpdir() failed");
2021 return true;
2022 }
2023 xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list);
2024 /* dummy for initialize all_charsets[] */
2025 get_charset_name(0);
2026
2027 srv_page_size = 0;
2028 srv_page_size_shift = 0;
2029 #ifdef BTR_CUR_HASH_ADAPT
2030 btr_ahi_parts = 1;
2031 #endif /* BTR_CUR_HASH_ADAPT */
2032
2033 if (innobase_page_size != (1LL << 14)) {
2034 size_t n_shift = get_bit_shift(size_t(innobase_page_size));
2035
2036 if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
2037 srv_page_size_shift = ulong(n_shift);
2038 srv_page_size = 1U << n_shift;
2039 msg("InnoDB: The universal page size of the "
2040 "database is set to %lu.", srv_page_size);
2041 } else {
2042 msg("invalid value of "
2043 "innobase_page_size: %lld", innobase_page_size);
2044 goto error;
2045 }
2046 } else {
2047 srv_page_size_shift = 14;
2048 srv_page_size = 1U << 14;
2049 }
2050
2051 /* Check that values don't overflow on 32-bit systems. */
2052 if (sizeof(ulint) == 4) {
2053 if (xtrabackup_use_memory > UINT_MAX32) {
2054 msg("mariabackup: use-memory can't be over 4GB"
2055 " on 32-bit systems");
2056 }
2057 }
2058
2059 static char default_path[2] = { FN_CURLIB, 0 };
2060 fil_path_to_mysql_datadir = default_path;
2061
2062 /* Set InnoDB initialization parameters according to the values
2063 read from MySQL .cnf file */
2064
2065 if (xtrabackup_backup) {
2066 msg("mariabackup: using the following InnoDB configuration:");
2067 } else {
2068 msg("mariabackup: using the following InnoDB configuration "
2069 "for recovery:");
2070 }
2071
2072 /*--------------- Data files -------------------------*/
2073
2074 /* The default dir for data files is the datadir of MySQL */
2075
2076 srv_data_home = (xtrabackup_backup && innobase_data_home_dir
2077 ? innobase_data_home_dir : default_path);
2078 msg("innodb_data_home_dir = %s", srv_data_home);
2079
2080 /* Set default InnoDB data file size to 10 MB and let it be
2081 auto-extending. Thus users can use InnoDB in >= 4.0 without having
2082 to specify any startup options. */
2083
2084 if (!innobase_data_file_path) {
2085 innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
2086 }
2087 msg("innodb_data_file_path = %s",
2088 innobase_data_file_path);
2089
2090 srv_sys_space.set_space_id(TRX_SYS_SPACE);
2091 srv_sys_space.set_name("innodb_system");
2092 srv_sys_space.set_path(srv_data_home);
2093 switch (srv_checksum_algorithm) {
2094 case SRV_CHECKSUM_ALGORITHM_FULL_CRC32:
2095 case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32:
2096 srv_sys_space.set_flags(FSP_FLAGS_FCRC32_MASK_MARKER
2097 | FSP_FLAGS_FCRC32_PAGE_SSIZE());
2098 break;
2099 default:
2100 srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE());
2101 }
2102
2103 if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
2104 goto error;
2105 }
2106
2107 srv_sys_space.normalize_size();
2108 srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);
2109
2110 /* -------------- Log files ---------------------------*/
2111
2112 /* The default dir for log files is the datadir of MySQL */
2113
2114 if (!(xtrabackup_backup && srv_log_group_home_dir)) {
2115 srv_log_group_home_dir = default_path;
2116 }
2117 if (xtrabackup_prepare && xtrabackup_incremental_dir) {
2118 srv_log_group_home_dir = xtrabackup_incremental_dir;
2119 }
2120 msg("innodb_log_group_home_dir = %s",
2121 srv_log_group_home_dir);
2122
2123 os_normalize_path(srv_log_group_home_dir);
2124
2125 if (strchr(srv_log_group_home_dir, ';')) {
2126 msg("syntax error in innodb_log_group_home_dir, ");
2127 goto error;
2128 }
2129
2130 srv_adaptive_flushing = FALSE;
2131
2132 /* We set srv_pool_size here in units of 1 kB. InnoDB internally
2133 changes the value so that it becomes the number of database pages. */
2134
2135 srv_buf_pool_size = (ulint) xtrabackup_use_memory;
2136 srv_buf_pool_chunk_unit = (ulong)srv_buf_pool_size;
2137 srv_buf_pool_instances = 1;
2138 srv_n_page_cleaners = 1;
2139
2140 srv_n_file_io_threads = (ulint) innobase_file_io_threads;
2141 srv_n_read_io_threads = (ulint) innobase_read_io_threads;
2142 srv_n_write_io_threads = (ulint) innobase_write_io_threads;
2143
2144 srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;
2145
2146 row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout;
2147
2148 srv_file_per_table = (my_bool) innobase_file_per_table;
2149
2150 srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog;
2151
2152 srv_max_n_open_files = ULINT_UNDEFINED - 5;
2153 srv_innodb_status = (ibool) innobase_create_status_file;
2154
2155 srv_print_verbose_log = verbose ? 2 : 1;
2156
2157 /* Store the default charset-collation number of this MySQL
2158 installation */
2159
2160 /* We cannot treat characterset here for now!! */
2161 data_mysql_default_charset_coll = (ulint)default_charset_info->number;
2162
2163 ut_ad(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number);
2164
2165 #ifdef _WIN32
2166 srv_use_native_aio = TRUE;
2167
2168 #elif defined(LINUX_NATIVE_AIO)
2169
2170 if (srv_use_native_aio) {
2171 msg("InnoDB: Using Linux native AIO");
2172 }
2173 #else
2174 /* Currently native AIO is supported only on windows and linux
2175 and that also when the support is compiled in. In all other
2176 cases, we ignore the setting of innodb_use_native_aio. */
2177 srv_use_native_aio = FALSE;
2178
2179 #endif
2180
2181 /* Assign the default value to srv_undo_dir if it's not specified, as
2182 my_getopt does not support default values for string options. We also
2183 ignore the option and override innodb_undo_directory on --prepare,
2184 because separate undo tablespaces are copied to the root backup
2185 directory. */
2186
2187 if (!srv_undo_dir || !xtrabackup_backup) {
2188 srv_undo_dir = (char*) ".";
2189 }
2190
2191 compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1);
2192
2193 /*
2194 * This option can be read both from the command line, and the
2195 * defaults file. The assignment should account for both cases,
2196 * and for "--innobackupex". Since the command line argument is
2197 * parsed after the defaults file, it takes precedence.
2198 */
2199 if (xtrabackup_innodb_force_recovery) {
2200 srv_force_recovery = xtrabackup_innodb_force_recovery;
2201 }
2202
2203 if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) {
2204 if (!xtrabackup_prepare) {
2205 msg("mariabackup: The option \"innodb_force_recovery\""
2206 " should only be used with \"%s\".",
2207 (innobackupex_mode ? "--apply-log" : "--prepare"));
2208 goto error;
2209 } else {
2210 msg("innodb_force_recovery = %lu", srv_force_recovery);
2211 }
2212 }
2213
2214 #ifdef _WIN32
2215 srv_use_native_aio = TRUE;
2216 #endif
2217 return false;
2218
2219 error:
2220 msg("mariabackup: innodb_init_param(): Error occurred.\n");
2221 return true;
2222 }
2223
innodb_init()2224 static bool innodb_init()
2225 {
2226 bool create_new_db = false;
2227 /* Check if the data files exist or not. */
2228 dberr_t err = srv_sys_space.check_file_spec(&create_new_db, 5U << 20);
2229
2230 if (err == DB_SUCCESS) {
2231 err = srv_start(create_new_db);
2232 }
2233
2234 if (err != DB_SUCCESS) {
2235 die("mariabackup: innodb_init() returned %d (%s).",
2236 err, ut_strerr(err));
2237 }
2238
2239 return(FALSE);
2240 }
2241
2242 /* ================= common ================= */
2243
2244 /***********************************************************************
2245 Read backup meta info.
2246 @return TRUE on success, FALSE on failure. */
2247 static
2248 my_bool
xtrabackup_read_metadata(char * filename)2249 xtrabackup_read_metadata(char *filename)
2250 {
2251 FILE *fp;
2252 my_bool r = TRUE;
2253
2254 fp = fopen(filename,"r");
2255 if(!fp) {
2256 msg("Error: cannot open %s", filename);
2257 return(FALSE);
2258 }
2259
2260 if (fscanf(fp, "backup_type = %29s\n", metadata_type)
2261 != 1) {
2262 r = FALSE;
2263 goto end;
2264 }
2265 /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
2266 format. */
2267 if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn)
2268 != 1) {
2269 r = FALSE;
2270 goto end;
2271 }
2272 if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn)
2273 != 1) {
2274 r = FALSE;
2275 goto end;
2276 }
2277 if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn)
2278 != 1) {
2279 metadata_last_lsn = 0;
2280 }
2281 /* Optional fields */
2282
2283 end:
2284 fclose(fp);
2285
2286 return(r);
2287 }
2288
2289 /***********************************************************************
2290 Print backup meta info to a specified buffer. */
2291 static
2292 void
xtrabackup_print_metadata(char * buf,size_t buf_len)2293 xtrabackup_print_metadata(char *buf, size_t buf_len)
2294 {
2295 /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
2296 format. */
2297 snprintf(buf, buf_len,
2298 "backup_type = %s\n"
2299 "from_lsn = " UINT64PF "\n"
2300 "to_lsn = " UINT64PF "\n"
2301 "last_lsn = " UINT64PF "\n",
2302 metadata_type,
2303 metadata_from_lsn,
2304 metadata_to_lsn,
2305 metadata_last_lsn);
2306 }
2307
2308 /***********************************************************************
2309 Stream backup meta info to a specified datasink.
2310 @return TRUE on success, FALSE on failure. */
2311 static
2312 my_bool
xtrabackup_stream_metadata(ds_ctxt_t * ds_ctxt)2313 xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt)
2314 {
2315 char buf[1024];
2316 size_t len;
2317 ds_file_t *stream;
2318 MY_STAT mystat;
2319 my_bool rc = TRUE;
2320
2321 xtrabackup_print_metadata(buf, sizeof(buf));
2322
2323 len = strlen(buf);
2324
2325 mystat.st_size = len;
2326 mystat.st_mtime = my_time(0);
2327
2328 stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat);
2329 if (stream == NULL) {
2330 msg("Error: cannot open output stream for %s", XTRABACKUP_METADATA_FILENAME);
2331 return(FALSE);
2332 }
2333
2334 if (ds_write(stream, buf, len)) {
2335 rc = FALSE;
2336 }
2337
2338 if (ds_close(stream)) {
2339 rc = FALSE;
2340 }
2341
2342 return(rc);
2343 }
2344
2345 /***********************************************************************
2346 Write backup meta info to a specified file.
2347 @return TRUE on success, FALSE on failure. */
2348 static
2349 my_bool
xtrabackup_write_metadata(const char * filepath)2350 xtrabackup_write_metadata(const char *filepath)
2351 {
2352 char buf[1024];
2353 size_t len;
2354 FILE *fp;
2355
2356 xtrabackup_print_metadata(buf, sizeof(buf));
2357
2358 len = strlen(buf);
2359
2360 fp = fopen(filepath, "w");
2361 if(!fp) {
2362 msg("Error: cannot open %s", filepath);
2363 return(FALSE);
2364 }
2365 if (fwrite(buf, len, 1, fp) < 1) {
2366 fclose(fp);
2367 return(FALSE);
2368 }
2369
2370 fclose(fp);
2371
2372 return(TRUE);
2373 }
2374
2375 /***********************************************************************
2376 Read meta info for an incremental delta.
2377 @return TRUE on success, FALSE on failure. */
2378 static my_bool
xb_read_delta_metadata(const char * filepath,xb_delta_info_t * info)2379 xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info)
2380 {
2381 FILE* fp;
2382 char key[51];
2383 char value[51];
2384 my_bool r = TRUE;
2385
2386 /* set defaults */
2387 ulint page_size = ULINT_UNDEFINED, zip_size = 0;
2388 info->space_id = ULINT_UNDEFINED;
2389
2390 fp = fopen(filepath, "r");
2391 if (!fp) {
2392 /* Meta files for incremental deltas are optional */
2393 return(TRUE);
2394 }
2395
2396 while (!feof(fp)) {
2397 if (fscanf(fp, "%50s = %50s\n", key, value) == 2) {
2398 if (strcmp(key, "page_size") == 0) {
2399 page_size = strtoul(value, NULL, 10);
2400 } else if (strcmp(key, "zip_size") == 0) {
2401 zip_size = strtoul(value, NULL, 10);
2402 } else if (strcmp(key, "space_id") == 0) {
2403 info->space_id = strtoul(value, NULL, 10);
2404 }
2405 }
2406 }
2407
2408 fclose(fp);
2409
2410 if (page_size == ULINT_UNDEFINED) {
2411 msg("page_size is required in %s", filepath);
2412 r = FALSE;
2413 } else {
2414 info->page_size = zip_size ? zip_size : page_size;
2415 }
2416
2417 if (info->space_id == ULINT_UNDEFINED) {
2418 msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 "
2419 "or earlier, some DDL operations between full and incremental "
2420 "backups may be handled incorrectly");
2421 }
2422
2423 return(r);
2424 }
2425
2426 /***********************************************************************
2427 Write meta info for an incremental delta.
2428 @return TRUE on success, FALSE on failure. */
2429 my_bool
xb_write_delta_metadata(const char * filename,const xb_delta_info_t * info)2430 xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info)
2431 {
2432 ds_file_t *f;
2433 char buf[64];
2434 my_bool ret;
2435 size_t len;
2436 MY_STAT mystat;
2437
2438 snprintf(buf, sizeof(buf),
2439 "page_size = " ULINTPF "\n"
2440 "zip_size = " ULINTPF " \n"
2441 "space_id = " ULINTPF "\n",
2442 info->page_size,
2443 info->zip_size,
2444 info->space_id);
2445 len = strlen(buf);
2446
2447 mystat.st_size = len;
2448 mystat.st_mtime = my_time(0);
2449
2450 f = ds_open(ds_meta, filename, &mystat);
2451 if (f == NULL) {
2452 msg("Error: Can't open output stream for %s",filename);
2453 return(FALSE);
2454 }
2455
2456 ret = (ds_write(f, buf, len) == 0);
2457
2458 if (ds_close(f)) {
2459 ret = FALSE;
2460 }
2461
2462 return(ret);
2463 }
2464
2465 /* ================= backup ================= */
2466 void
xtrabackup_io_throttling(void)2467 xtrabackup_io_throttling(void)
2468 {
2469 if (xtrabackup_backup && xtrabackup_throttle && (io_ticket--) < 0) {
2470 os_event_reset(wait_throttle);
2471 os_event_wait(wait_throttle);
2472 }
2473 }
2474
2475 static
regex_list_check_match(const regex_list_t & list,const char * name)2476 my_bool regex_list_check_match(
2477 const regex_list_t& list,
2478 const char* name)
2479 {
2480 regmatch_t tables_regmatch[1];
2481 for (regex_list_t::const_iterator i = list.begin(), end = list.end();
2482 i != end; ++i) {
2483 const regex_t& regex = *i;
2484 int regres = regexec(®ex, name, 1, tables_regmatch, 0);
2485
2486 if (regres != REG_NOMATCH) {
2487 return(TRUE);
2488 }
2489 }
2490 return(FALSE);
2491 }
2492
2493 static
2494 my_bool
find_filter_in_hashtable(const char * name,hash_table_t * table,xb_filter_entry_t ** result)2495 find_filter_in_hashtable(
2496 const char* name,
2497 hash_table_t* table,
2498 xb_filter_entry_t** result
2499 )
2500 {
2501 xb_filter_entry_t* found = NULL;
2502 HASH_SEARCH(name_hash, table, ut_fold_string(name),
2503 xb_filter_entry_t*,
2504 found, (void) 0,
2505 !strcmp(found->name, name));
2506
2507 if (found && result) {
2508 *result = found;
2509 }
2510 return (found != NULL);
2511 }
2512
2513 /************************************************************************
2514 Checks if a given table name matches any of specifications given in
2515 regex_list or tables_hash.
2516
2517 @return TRUE on match or both regex_list and tables_hash are empty.*/
2518 static my_bool
check_if_table_matches_filters(const char * name,const regex_list_t & regex_list,hash_table_t * tables_hash)2519 check_if_table_matches_filters(const char *name,
2520 const regex_list_t& regex_list,
2521 hash_table_t* tables_hash)
2522 {
2523 if (regex_list.empty() && !tables_hash) {
2524 return(FALSE);
2525 }
2526
2527 if (regex_list_check_match(regex_list, name)) {
2528 return(TRUE);
2529 }
2530
2531 if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
2532 return(TRUE);
2533 }
2534
2535 return FALSE;
2536 }
2537
2538 enum skip_database_check_result {
2539 DATABASE_SKIP,
2540 DATABASE_SKIP_SOME_TABLES,
2541 DATABASE_DONT_SKIP,
2542 DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
2543 };
2544
2545 /************************************************************************
2546 Checks if a database specified by name should be skipped from backup based on
2547 the --databases, --databases_file or --databases_exclude options.
2548
2549 @return TRUE if entire database should be skipped,
2550 FALSE otherwise.
2551 */
2552 static
2553 skip_database_check_result
check_if_skip_database(const char * name)2554 check_if_skip_database(
2555 const char* name /*!< in: path to the database */
2556 )
2557 {
2558 /* There are some filters for databases, check them */
2559 xb_filter_entry_t* database = NULL;
2560
2561 if (databases_exclude_hash &&
2562 find_filter_in_hashtable(name, databases_exclude_hash,
2563 &database) &&
2564 (!database->has_tables || !databases_include_hash)) {
2565 /* Database is found and there are no tables specified,
2566 skip entire db. */
2567 return DATABASE_SKIP;
2568 }
2569
2570 if (databases_include_hash) {
2571 if (!find_filter_in_hashtable(name, databases_include_hash,
2572 &database)) {
2573 /* Database isn't found, skip the database */
2574 return DATABASE_SKIP;
2575 } else if (database->has_tables) {
2576 return DATABASE_SKIP_SOME_TABLES;
2577 } else {
2578 return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
2579 }
2580 }
2581
2582 return DATABASE_DONT_SKIP;
2583 }
2584
2585 /************************************************************************
2586 Checks if a database specified by path should be skipped from backup based on
2587 the --databases, --databases_file or --databases_exclude options.
2588
2589 @return TRUE if the table should be skipped. */
2590 my_bool
check_if_skip_database_by_path(const char * path)2591 check_if_skip_database_by_path(
2592 const char* path /*!< in: path to the db directory. */
2593 )
2594 {
2595 if (databases_include_hash == NULL &&
2596 databases_exclude_hash == NULL) {
2597 return(FALSE);
2598 }
2599
2600 const char* db_name = strrchr(path, OS_PATH_SEPARATOR);
2601 if (db_name == NULL) {
2602 db_name = path;
2603 } else {
2604 ++db_name;
2605 }
2606
2607 return check_if_skip_database(db_name) == DATABASE_SKIP;
2608 }
2609
2610 /************************************************************************
2611 Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
2612 or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on
2613 the --tables or --tables-file options.
2614
2615 @return TRUE if the table should be skipped. */
2616 my_bool
check_if_skip_table(const char * name)2617 check_if_skip_table(
2618 /******************/
2619 const char* name) /*!< in: path to the table */
2620 {
2621 char buf[FN_REFLEN];
2622 const char *dbname, *tbname;
2623 const char *ptr;
2624 char *eptr;
2625
2626
2627 dbname = NULL;
2628 tbname = name;
2629 while ((ptr = strchr(tbname, '/')) != NULL) {
2630 dbname = tbname;
2631 tbname = ptr + 1;
2632 }
2633
2634 if (strncmp(tbname, tmp_file_prefix, tmp_file_prefix_length) == 0) {
2635 return TRUE;
2636 }
2637
2638 if (regex_exclude_list.empty() &&
2639 regex_include_list.empty() &&
2640 tables_include_hash == NULL &&
2641 tables_exclude_hash == NULL &&
2642 databases_include_hash == NULL &&
2643 databases_exclude_hash == NULL) {
2644 return(FALSE);
2645 }
2646
2647 if (dbname == NULL) {
2648 return(FALSE);
2649 }
2650
2651 strncpy(buf, dbname, FN_REFLEN - 1);
2652 buf[FN_REFLEN - 1] = '\0';
2653 buf[tbname - 1 - dbname] = '\0';
2654
2655 const skip_database_check_result skip_database =
2656 check_if_skip_database(buf);
2657 if (skip_database == DATABASE_SKIP) {
2658 return (TRUE);
2659 }
2660
2661 buf[tbname - 1 - dbname] = '.';
2662
2663 /* Check if there's a suffix in the table name. If so, truncate it. We
2664 rely on the fact that a dot cannot be a part of a table name (it is
2665 encoded by the server with the @NNNN syntax). */
2666 if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) {
2667
2668 *eptr = '\0';
2669 }
2670
2671 /* For partitioned tables first try to match against the regexp
2672 without truncating the #P#... suffix so we can backup individual
2673 partitions with regexps like '^test[.]t#P#p5' */
2674 if (check_if_table_matches_filters(buf, regex_exclude_list,
2675 tables_exclude_hash)) {
2676 return(TRUE);
2677 }
2678 if (check_if_table_matches_filters(buf, regex_include_list,
2679 tables_include_hash)) {
2680 return(FALSE);
2681 }
2682 if ((eptr = strstr(buf, "#P#")) != NULL) {
2683 *eptr = 0;
2684
2685 if (check_if_table_matches_filters(buf, regex_exclude_list,
2686 tables_exclude_hash)) {
2687 return (TRUE);
2688 }
2689 if (check_if_table_matches_filters(buf, regex_include_list,
2690 tables_include_hash)) {
2691 return(FALSE);
2692 }
2693 }
2694
2695 if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
2696 /* Database is in include-list, and qualified name wasn't
2697 found in any of exclusion filters.*/
2698 return (FALSE);
2699 }
2700
2701 if (skip_database == DATABASE_SKIP_SOME_TABLES ||
2702 !regex_include_list.empty() ||
2703 tables_include_hash) {
2704
2705 /* Include lists are present, but qualified name
2706 failed to match any.*/
2707 return(TRUE);
2708 }
2709
2710 return(FALSE);
2711 }
2712
2713 const char*
xb_get_copy_action(const char * dflt)2714 xb_get_copy_action(const char *dflt)
2715 {
2716 const char *action;
2717
2718 if (xtrabackup_stream) {
2719 if (xtrabackup_compress) {
2720 action = "Compressing and streaming";
2721 } else {
2722 action = "Streaming";
2723 }
2724 } else {
2725 if (xtrabackup_compress) {
2726 action = "Compressing";
2727 } else {
2728 action = dflt;
2729 }
2730 }
2731
2732 return(action);
2733 }
2734
2735
2736 /** Copy innodb data file to the specified destination.
2737
2738 @param[in] node file node of a tablespace
2739 @param[in] thread_n thread id, used in the text of diagnostic messages
2740 @param[in] dest_name destination file name
2741 @param[in] write_filter write filter to copy data, can be pass-through filter
2742 for full backup, pages filter for incremental backup, etc.
2743
2744 @return FALSE on success and TRUE on error */
xtrabackup_copy_datafile(fil_node_t * node,uint thread_n,const char * dest_name,const xb_write_filt_t & write_filter,CorruptedPages & corrupted_pages)2745 static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n,
2746 const char *dest_name,
2747 const xb_write_filt_t &write_filter,
2748 CorruptedPages &corrupted_pages)
2749 {
2750 char dst_name[FN_REFLEN];
2751 ds_file_t *dstfile = NULL;
2752 xb_fil_cur_t cursor;
2753 xb_fil_cur_result_t res;
2754 xb_write_filt_ctxt_t write_filt_ctxt;
2755 const char *action;
2756 xb_read_filt_t *read_filter;
2757 my_bool rc = FALSE;
2758
2759 /* Get the name and the path for the tablespace. node->name always
2760 contains the path (which may be absolute for remote tablespaces in
2761 5.6+). space->name contains the tablespace name in the form
2762 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
2763 multi-node shared tablespace, space->name contains the name of the first
2764 node, but that's irrelevant, since we only need node_name to match them
2765 against filters, and the shared tablespace is always copied regardless
2766 of the filters value. */
2767
2768 const char* const node_name = node->space->name;
2769 const char* const node_path = node->name;
2770
2771 if (fil_is_user_tablespace_id(node->space->id)
2772 && check_if_skip_table(node_name)) {
2773 msg(thread_n, "Skipping %s.", node_name);
2774 return(FALSE);
2775 }
2776
2777 memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
2778
2779 bool was_dropped;
2780 pthread_mutex_lock(&backup_mutex);
2781 was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end());
2782 pthread_mutex_unlock(&backup_mutex);
2783 if (was_dropped) {
2784 if (node->is_open()) {
2785 mutex_enter(&fil_system.mutex);
2786 node->close();
2787 mutex_exit(&fil_system.mutex);
2788 }
2789 goto skip;
2790 }
2791
2792 if (!changed_page_bitmap) {
2793 read_filter = &rf_pass_through;
2794 }
2795 else {
2796 read_filter = &rf_bitmap;
2797 }
2798
2799 res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX);
2800 if (res == XB_FIL_CUR_SKIP) {
2801 goto skip;
2802 } else if (res == XB_FIL_CUR_ERROR) {
2803 goto error;
2804 }
2805
2806 strncpy(dst_name, dest_name ? dest_name : cursor.rel_path,
2807 sizeof dst_name - 1);
2808 dst_name[sizeof dst_name - 1] = '\0';
2809
2810 ut_a(write_filter.process != NULL);
2811
2812 if (write_filter.init != NULL &&
2813 !write_filter.init(&write_filt_ctxt, dst_name, &cursor,
2814 opt_log_innodb_page_corruption ? &corrupted_pages : NULL)) {
2815 msg (thread_n, "mariabackup: error: failed to initialize page write filter.");
2816 goto error;
2817 }
2818
2819 dstfile = ds_open(ds_data, dst_name, &cursor.statinfo);
2820 if (dstfile == NULL) {
2821 msg(thread_n,"mariabackup: error: can't open the destination stream for %s", dst_name);
2822 goto error;
2823 }
2824
2825 action = xb_get_copy_action();
2826
2827 if (xtrabackup_stream) {
2828 msg(thread_n, "%s %s", action, node_path);
2829 } else {
2830 msg(thread_n, "%s %s to %s", action, node_path, dstfile->path);
2831 }
2832
2833 /* The main copy loop */
2834 while ((res = xb_fil_cur_read(&cursor, corrupted_pages)) ==
2835 XB_FIL_CUR_SUCCESS) {
2836 if (!write_filter.process(&write_filt_ctxt, dstfile)) {
2837 goto error;
2838 }
2839 }
2840
2841 if (res == XB_FIL_CUR_ERROR) {
2842 goto error;
2843 }
2844
2845 if (write_filter.finalize
2846 && !write_filter.finalize(&write_filt_ctxt, dstfile)) {
2847 goto error;
2848 }
2849
2850 pthread_mutex_lock(&backup_mutex);
2851 ddl_tracker.tables_in_backup[node->space->id] = node_name;
2852 pthread_mutex_unlock(&backup_mutex);
2853
2854 /* close */
2855 msg(thread_n," ...done");
2856 xb_fil_cur_close(&cursor);
2857 if (ds_close(dstfile)) {
2858 rc = TRUE;
2859 }
2860 if (write_filter.deinit) {
2861 write_filter.deinit(&write_filt_ctxt);
2862 }
2863 return(rc);
2864
2865 error:
2866 xb_fil_cur_close(&cursor);
2867 if (dstfile != NULL) {
2868 ds_close(dstfile);
2869 }
2870 if (write_filter.deinit) {
2871 write_filter.deinit(&write_filt_ctxt);;
2872 }
2873 msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed.");
2874 return(TRUE); /*ERROR*/
2875
2876 skip:
2877
2878 if (dstfile != NULL) {
2879 ds_close(dstfile);
2880 }
2881 if (write_filter.deinit) {
2882 write_filter.deinit(&write_filt_ctxt);
2883 }
2884 msg(thread_n,"Warning: We assume the table was dropped during xtrabackup execution and ignore the tablespace %s", node_name);
2885 return(FALSE);
2886 }
2887
2888 /** Copy redo log blocks to the data sink.
2889 @param start_lsn buffer start LSN
2890 @param end_lsn buffer end LSN
2891 @param last whether we are copying the final part of the log
2892 @return last scanned LSN
2893 @retval 0 on failure */
xtrabackup_copy_log(lsn_t start_lsn,lsn_t end_lsn,bool last)2894 static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last)
2895 {
2896 lsn_t scanned_lsn = start_lsn;
2897 const byte* log_block = log_sys.buf;
2898 bool more_data = false;
2899
2900 for (ulint scanned_checkpoint = 0;
2901 scanned_lsn < end_lsn;
2902 log_block += OS_FILE_LOG_BLOCK_SIZE) {
2903 ulint checkpoint = log_block_get_checkpoint_no(log_block);
2904
2905 if (scanned_checkpoint > checkpoint
2906 && scanned_checkpoint - checkpoint >= 0x80000000UL) {
2907 /* Garbage from a log buffer flush which was made
2908 before the most recent database recovery */
2909 msg(0,"checkpoint wrap: " LSN_PF ",%zx,%zx",
2910 scanned_lsn, scanned_checkpoint, checkpoint);
2911 break;
2912 }
2913
2914 scanned_checkpoint = checkpoint;
2915
2916 ulint data_len = log_block_get_data_len(log_block);
2917
2918 more_data = recv_sys_add_to_parsing_buf(
2919 log_block,
2920 scanned_lsn + data_len);
2921
2922 recv_sys.scanned_lsn = scanned_lsn + data_len;
2923
2924 if (data_len == OS_FILE_LOG_BLOCK_SIZE) {
2925 /* We got a full log block. */
2926 scanned_lsn += data_len;
2927 } else if (data_len >= log_sys.trailer_offset()
2928 || data_len < LOG_BLOCK_HDR_SIZE) {
2929 /* We got a garbage block (abrupt end of the log). */
2930 msg(0,"garbage block: " LSN_PF ",%zu",scanned_lsn, data_len);
2931 break;
2932 } else {
2933 /* We got a partial block (abrupt end of the log). */
2934 scanned_lsn += data_len;
2935 break;
2936 }
2937 }
2938
2939 store_t store = STORE_NO;
2940
2941 if (more_data && recv_parse_log_recs(0, &store, 0, false)) {
2942
2943 msg("Error: copying the log failed");
2944
2945 return(0);
2946 }
2947
2948 recv_sys_justify_left_parsing_buf();
2949
2950 log_sys.log.scanned_lsn = scanned_lsn;
2951
2952 end_lsn = last
2953 ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE)
2954 : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1);
2955
2956 if (ulint write_size = ulint(end_lsn - start_lsn)) {
2957 if (srv_encrypt_log) {
2958 log_crypt(log_sys.buf, start_lsn, write_size);
2959 }
2960
2961 if (ds_write(dst_log_file, log_sys.buf, write_size)) {
2962 msg("Error: write to logfile failed");
2963 return(0);
2964 }
2965 }
2966
2967 return(scanned_lsn);
2968 }
2969
2970 /** Copy redo log until the current end of the log is reached
2971 @param last whether we are copying the final part of the log
2972 @return whether the operation failed */
xtrabackup_copy_logfile(bool last=false)2973 static bool xtrabackup_copy_logfile(bool last = false)
2974 {
2975 ut_a(dst_log_file != NULL);
2976 ut_ad(recv_sys.is_initialised());
2977
2978 bool overwritten_block = false;
2979 lsn_t start_lsn;
2980 lsn_t end_lsn;
2981
2982 recv_sys.parse_start_lsn = log_copy_scanned_lsn;
2983 recv_sys.scanned_lsn = log_copy_scanned_lsn;
2984
2985 start_lsn = ut_uint64_align_down(log_copy_scanned_lsn,
2986 OS_FILE_LOG_BLOCK_SIZE);
2987 do {
2988 end_lsn = start_lsn + RECV_SCAN_SIZE;
2989
2990 xtrabackup_io_throttling();
2991
2992 log_mutex_enter();
2993 lsn_t lsn= start_lsn;
2994 for (int retries= 0; retries < 100; retries++) {
2995 if (log_sys.log.read_log_seg(&lsn, end_lsn)
2996 || lsn != start_lsn) {
2997 break;
2998 }
2999 msg("Retrying read of log at LSN=" LSN_PF, lsn);
3000 my_sleep(1000);
3001 }
3002
3003 if (lsn == start_lsn) {
3004 overwritten_block= !recv_sys.found_corrupt_log
3005 && (innodb_log_checksums || log_sys.log.is_encrypted())
3006 && log_block_calc_checksum_crc32(log_sys.buf) ==
3007 log_block_get_checksum(log_sys.buf)
3008 && log_block_get_hdr_no(log_sys.buf) >
3009 log_block_convert_lsn_to_no(start_lsn);
3010 start_lsn = 0;
3011 } else {
3012 mutex_enter(&recv_sys.mutex);
3013 start_lsn = xtrabackup_copy_log(start_lsn, lsn, last);
3014 mutex_exit(&recv_sys.mutex);
3015 }
3016
3017 log_mutex_exit();
3018
3019 if (!start_lsn) {
3020 const char *reason = recv_sys.found_corrupt_log
3021 ? "corrupt log."
3022 : (overwritten_block
3023 ? "redo log block is overwritten, please increase redo log size with innodb_log_file_size parameter."
3024 : ((innodb_log_checksums || log_sys.log.is_encrypted())
3025 ? "redo log block checksum does not match."
3026 : "unknown reason as innodb_log_checksums is switched off and redo"
3027 " log is not encrypted."));
3028 die("xtrabackup_copy_logfile() failed: %s", reason);
3029 return true;
3030 }
3031 } while (start_lsn == end_lsn);
3032
3033 ut_ad(start_lsn == log_sys.log.scanned_lsn);
3034
3035 msg(">> log scanned up to (" LSN_PF ")", start_lsn);
3036
3037 /* update global variable*/
3038 pthread_mutex_lock(&backup_mutex);
3039 log_copy_scanned_lsn = start_lsn;
3040 pthread_cond_broadcast(&scanned_lsn_cond);
3041 pthread_mutex_unlock(&backup_mutex);
3042 return(false);
3043 }
3044
3045 /**
3046 Wait until redo log copying thread processes given lsn
3047 */
backup_wait_for_lsn(lsn_t lsn)3048 void backup_wait_for_lsn(lsn_t lsn) {
3049 bool completed = false;
3050 pthread_mutex_lock(&backup_mutex);
3051 do {
3052 pthread_cond_wait(&scanned_lsn_cond, &backup_mutex);
3053 completed = log_copy_scanned_lsn >= lsn;
3054 } while (!completed);
3055 pthread_mutex_unlock(&backup_mutex);
3056 }
3057
3058 extern lsn_t server_lsn_after_lock;
3059
DECLARE_THREAD(log_copying_thread)3060 static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
3061 {
3062 /*
3063 Initialize mysys thread-specific memory so we can
3064 use mysys functions in this thread.
3065 */
3066 my_thread_init();
3067
3068 for (;;) {
3069 os_event_reset(log_copying_stop);
3070 os_event_wait_time_low(log_copying_stop,
3071 xtrabackup_log_copy_interval * 1000ULL,
3072 0);
3073 if (xtrabackup_copy_logfile()) {
3074 break;
3075 }
3076
3077 log_mutex_enter();
3078 bool completed = metadata_to_lsn
3079 && metadata_to_lsn <= log_copy_scanned_lsn;
3080 log_mutex_exit();
3081 if (completed) {
3082 break;
3083 }
3084 }
3085
3086 log_copying_running = false;
3087 my_thread_end();
3088 os_thread_exit();
3089
3090 return(0);
3091 }
3092
3093 /* io throttle watching (rough) */
DECLARE_THREAD(io_watching_thread)3094 static os_thread_ret_t DECLARE_THREAD(io_watching_thread)(void*)
3095 {
3096 /* currently, for --backup only */
3097 ut_a(xtrabackup_backup);
3098
3099 while (log_copying_running && !metadata_to_lsn) {
3100 os_thread_sleep(1000000); /*1 sec*/
3101 io_ticket = xtrabackup_throttle;
3102 os_event_set(wait_throttle);
3103 }
3104
3105 /* stop io throttle */
3106 xtrabackup_throttle = 0;
3107 os_event_set(wait_throttle);
3108
3109 io_watching_thread_running = false;
3110
3111 os_thread_exit();
3112
3113 return(0);
3114 }
3115
3116 #ifndef DBUG_OFF
dbug_mariabackup_get_val(const char * event,const char * key)3117 char *dbug_mariabackup_get_val(const char *event, const char *key)
3118 {
3119 char envvar[FN_REFLEN];
3120 if (key) {
3121 snprintf(envvar, sizeof(envvar), "%s_%s", event, key);
3122 char *slash = strchr(envvar, '/');
3123 if (slash)
3124 *slash = '_';
3125 } else {
3126 strncpy(envvar, event, sizeof envvar - 1);
3127 envvar[sizeof envvar - 1] = '\0';
3128 }
3129 return getenv(envvar);
3130 }
3131
3132 /*
3133 In debug mode, execute SQL statement that was passed via environment.
3134 To use this facility, you need to
3135
3136 1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key););
3137 to the code. key is usually a table name
3138 2. Set environment variable my_event_name_$key SQL statement you want to execute
3139 when event occurs, in DBUG_EXECUTE_IF from above.
3140 In mtr , you can set environment via 'let' statement (do not use $ as the first char
3141 for the variable)
3142 3. start mariabackup with --dbug=+d,debug_mariabackup_events
3143 */
dbug_mariabackup_event(const char * event,const char * key)3144 void dbug_mariabackup_event(const char *event,const char *key)
3145 {
3146 char *sql = dbug_mariabackup_get_val(event, key);
3147 if (sql && *sql) {
3148 msg("dbug_mariabackup_event : executing '%s'", sql);
3149 xb_mysql_query(mysql_connection, sql, false, true);
3150 }
3151 }
3152 #endif // DBUG_OFF
3153
3154 /**************************************************************************
3155 Datafiles copying thread.*/
3156 static
3157 os_thread_ret_t
DECLARE_THREAD(data_copy_thread_func)3158 DECLARE_THREAD(data_copy_thread_func)(
3159 /*==================*/
3160 void *arg) /* thread context */
3161 {
3162 data_thread_ctxt_t *ctxt = (data_thread_ctxt_t *) arg;
3163 uint num = ctxt->num;
3164 fil_node_t* node;
3165 ut_ad(ctxt->corrupted_pages);
3166
3167 /*
3168 Initialize mysys thread-specific memory so we can
3169 use mysys functions in this thread.
3170 */
3171 my_thread_init();
3172
3173 while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
3174 DBUG_MARIABACKUP_EVENT("before_copy", node->space->name);
3175 DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", node->space->name,
3176 backup_wait_for_lsn(get_current_lsn(mysql_connection)););
3177 /* copy the datafile */
3178 if (xtrabackup_copy_datafile(node, num, NULL,
3179 xtrabackup_incremental ? wf_incremental : wf_write_through,
3180 *ctxt->corrupted_pages))
3181 die("failed to copy datafile.");
3182
3183 DBUG_MARIABACKUP_EVENT("after_copy", node->space->name);
3184
3185 }
3186
3187 pthread_mutex_lock(ctxt->count_mutex);
3188 (*ctxt->count)--;
3189 pthread_mutex_unlock(ctxt->count_mutex);
3190
3191 my_thread_end();
3192 os_thread_exit();
3193 OS_THREAD_DUMMY_RETURN;
3194 }
3195
3196 /************************************************************************
3197 Initialize the appropriate datasink(s). Both local backups and streaming in the
3198 'xbstream' format allow parallel writes so we can write directly.
3199
3200 Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks
3201 for the data stream (and don't allow parallel data copying) and for metainfo
3202 files (including ib_logfile0). The second datasink writes to temporary
3203 files first, and then streams them in a serialized way when closed. */
3204 static void
xtrabackup_init_datasinks(void)3205 xtrabackup_init_datasinks(void)
3206 {
3207 /* Start building out the pipelines from the terminus back */
3208 if (xtrabackup_stream) {
3209 /* All streaming goes to stdout */
3210 ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
3211 DS_TYPE_STDOUT);
3212 } else {
3213 /* Local filesystem */
3214 ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
3215 DS_TYPE_LOCAL);
3216 }
3217
3218 /* Track it for destruction */
3219 xtrabackup_add_datasink(ds_data);
3220
3221 /* Stream formatting */
3222 if (xtrabackup_stream) {
3223 ds_ctxt_t *ds;
3224
3225 ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM);
3226 ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM);
3227
3228 xtrabackup_add_datasink(ds);
3229
3230 ds_set_pipe(ds, ds_data);
3231 ds_data = ds;
3232
3233
3234 ds_redo = ds_meta = ds_data;
3235 }
3236
3237 /* Compression for ds_data and ds_redo */
3238 if (xtrabackup_compress) {
3239 ds_ctxt_t *ds;
3240
3241 /* Use a 1 MB buffer for compressed output stream */
3242 ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
3243 ds_buffer_set_size(ds, 1024 * 1024);
3244 xtrabackup_add_datasink(ds);
3245 ds_set_pipe(ds, ds_data);
3246 if (ds_data != ds_redo) {
3247 ds_data = ds;
3248 ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
3249 ds_buffer_set_size(ds, 1024 * 1024);
3250 xtrabackup_add_datasink(ds);
3251 ds_set_pipe(ds, ds_redo);
3252 ds_redo = ds;
3253 } else {
3254 ds_redo = ds_data = ds;
3255 }
3256
3257 ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
3258 xtrabackup_add_datasink(ds);
3259 ds_set_pipe(ds, ds_data);
3260 if (ds_data != ds_redo) {
3261 ds_data = ds;
3262 ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
3263 xtrabackup_add_datasink(ds);
3264 ds_set_pipe(ds, ds_redo);
3265 ds_redo = ds;
3266 } else {
3267 ds_redo = ds_data = ds;
3268 }
3269 }
3270 }
3271
3272 /************************************************************************
3273 Destroy datasinks.
3274
3275 Destruction is done in the specific order to not violate their order in the
3276 pipeline so that each datasink is able to flush data down the pipeline. */
xtrabackup_destroy_datasinks(void)3277 static void xtrabackup_destroy_datasinks(void)
3278 {
3279 for (uint i = actual_datasinks; i > 0; i--) {
3280 ds_destroy(datasinks[i-1]);
3281 datasinks[i-1] = NULL;
3282 }
3283 ds_data = NULL;
3284 ds_meta = NULL;
3285 ds_redo = NULL;
3286 }
3287
3288 #define SRV_MAX_N_PENDING_SYNC_IOS 100
3289
3290 /** Initialize the tablespace cache subsystem. */
3291 static
3292 void
xb_fil_io_init()3293 xb_fil_io_init()
3294 {
3295 fil_system.create(srv_file_per_table ? 50000 : 5000);
3296 }
3297
3298 static
3299 Datafile*
xb_new_datafile(const char * name,bool is_remote)3300 xb_new_datafile(const char *name, bool is_remote)
3301 {
3302 if (is_remote) {
3303 RemoteDatafile *remote_file = new RemoteDatafile();
3304 remote_file->set_name(name);
3305 return(remote_file);
3306 } else {
3307 Datafile *file = new Datafile();
3308 file->set_name(name);
3309 file->make_filepath(".", name, IBD);
3310 return(file);
3311 }
3312 }
3313
3314
3315 /** Load tablespace.
3316
3317 @param[in] dirname directory name of the tablespace to open
3318 @param[in] filname file name of the tablespece to open
3319 @param[in] is_remote true if tablespace file is .isl
3320 @param[in] set_size true if we need to set tablespace size in pages explixitly.
3321 If this parameter is set, the size and free pages limit will not be read
3322 from page 0.
3323 */
xb_load_single_table_tablespace(const char * dirname,const char * filname,bool is_remote,bool set_size)3324 static void xb_load_single_table_tablespace(const char *dirname,
3325 const char *filname,
3326 bool is_remote, bool set_size)
3327 {
3328 ut_ad(srv_operation == SRV_OPERATION_BACKUP
3329 || srv_operation == SRV_OPERATION_RESTORE_DELTA
3330 || srv_operation == SRV_OPERATION_RESTORE);
3331 /* Ignore .isl files on XtraBackup recovery. All tablespaces must be
3332 local. */
3333 if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) {
3334 return;
3335 }
3336 if (check_if_skip_table(filname)) {
3337 return;
3338 }
3339
3340 /* The name ends in .ibd or .isl;
3341 try opening the file */
3342 char* name;
3343 size_t dirlen = dirname == NULL ? 0 : strlen(dirname);
3344 size_t namelen = strlen(filname);
3345 ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2;
3346 lsn_t flush_lsn;
3347 dberr_t err;
3348 fil_space_t *space;
3349
3350 name = static_cast<char*>(ut_malloc_nokey(pathlen));
3351
3352 if (dirname != NULL) {
3353 snprintf(name, pathlen, "%s/%s", dirname, filname);
3354 name[pathlen - 5] = 0;
3355 } else {
3356 snprintf(name, pathlen, "%s", filname);
3357 name[pathlen - 5] = 0;
3358 }
3359
3360 Datafile *file = xb_new_datafile(name, is_remote);
3361
3362 if (file->open_read_only(true) != DB_SUCCESS) {
3363 die("Can't open datafile %s", name);
3364 }
3365
3366 for (int i = 0; i < 10; i++) {
3367 err = file->validate_first_page(&flush_lsn);
3368 if (err != DB_CORRUPTION) {
3369 break;
3370 }
3371
3372 my_sleep(1000);
3373 }
3374
3375 bool is_empty_file = file->exists() && file->is_empty_file();
3376
3377 if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
3378 os_offset_t n_pages = 0;
3379 if (set_size) {
3380 os_offset_t node_size = os_file_get_size(file->handle());
3381 ut_a(node_size != (os_offset_t) -1);
3382 n_pages = node_size / fil_space_t::physical_size(file->flags());
3383 }
3384 space = fil_space_create(
3385 name, file->space_id(), file->flags(),
3386 FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */);
3387
3388 ut_a(space != NULL);
3389
3390 space->add(file->filepath(), OS_FILE_CLOSED, ulint(n_pages),
3391 false, false);
3392 /* by opening the tablespace we forcing node and space objects
3393 in the cache to be populated with fields from space header */
3394 space->open();
3395
3396 if (srv_operation == SRV_OPERATION_RESTORE_DELTA
3397 || xb_close_files) {
3398 space->close();
3399 }
3400 }
3401
3402 delete file;
3403
3404 if (err != DB_SUCCESS && xtrabackup_backup && !is_empty_file) {
3405 die("Failed to validate first page of the file %s, error %d",name, (int)err);
3406 }
3407
3408 ut_free(name);
3409 }
3410
xb_load_single_table_tablespace(const std::string & space_name,bool set_size)3411 static void xb_load_single_table_tablespace(const std::string &space_name,
3412 bool set_size)
3413 {
3414 std::string name(space_name);
3415 bool is_remote= access((name + ".ibd").c_str(), R_OK) != 0;
3416 const char *extension= is_remote ? ".isl" : ".ibd";
3417 name.append(extension);
3418 char buf[FN_REFLEN];
3419 strncpy(buf, name.c_str(), sizeof buf - 1);
3420 buf[sizeof buf - 1]= '\0';
3421 const char *dbname= buf;
3422 char *p= strchr(buf, '/');
3423 if (p == 0)
3424 die("Unexpected tablespace %s filename %s", space_name.c_str(),
3425 name.c_str());
3426 ut_a(p);
3427 *p= 0;
3428 const char *tablename= p + 1;
3429 xb_load_single_table_tablespace(dbname, tablename, is_remote, set_size);
3430 }
3431
3432 #ifdef _WIN32
3433 /**
3434 The os_file_opendir() function opens a directory stream corresponding to the
3435 directory named by the dirname argument. The directory stream is positioned
3436 at the first entry. In both Unix and Windows we automatically skip the '.'
3437 and '..' items at the start of the directory listing.
3438 @param[in] dirname directory name; it must not contain a trailing
3439 '\' or '/'
3440 @return directory stream, NULL if error */
os_file_opendir(const char * dirname)3441 os_file_dir_t os_file_opendir(const char *dirname)
3442 {
3443 char path[OS_FILE_MAX_PATH + 3];
3444
3445 ut_a(strlen(dirname) < OS_FILE_MAX_PATH);
3446
3447 strcpy(path, dirname);
3448 strcpy(path + strlen(path), "\\*");
3449
3450 /* Note that in Windows opening the 'directory stream' also retrieves
3451 the first entry in the directory. Since it is '.', that is no problem,
3452 as we will skip over the '.' and '..' entries anyway. */
3453
3454 LPWIN32_FIND_DATA lpFindFileData= static_cast<LPWIN32_FIND_DATA>
3455 (ut_malloc_nokey(sizeof(WIN32_FIND_DATA)));
3456 os_file_dir_t dir= FindFirstFile((LPCTSTR) path, lpFindFileData);
3457 ut_free(lpFindFileData);
3458
3459 return dir;
3460 }
3461 #endif
3462
3463 /** This function returns information of the next file in the directory. We jump
3464 over the '.' and '..' entries in the directory.
3465 @param[in] dirname directory name or path
3466 @param[in] dir directory stream
3467 @param[out] info buffer where the info is returned
3468 @return 0 if ok, -1 if error, 1 if at the end of the directory */
3469 int
os_file_readdir_next_file(const char * dirname,os_file_dir_t dir,os_file_stat_t * info)3470 os_file_readdir_next_file(
3471 const char* dirname,
3472 os_file_dir_t dir,
3473 os_file_stat_t* info)
3474 {
3475 #ifdef _WIN32
3476 BOOL ret;
3477 int status;
3478 WIN32_FIND_DATA find_data;
3479
3480 next_file:
3481 ret = FindNextFile(dir, &find_data);
3482
3483 if (ret > 0) {
3484
3485 const char* name;
3486
3487 name = static_cast<const char*>(find_data.cFileName);
3488
3489 ut_a(strlen(name) < OS_FILE_MAX_PATH);
3490
3491 if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
3492
3493 goto next_file;
3494 }
3495
3496 strcpy(info->name, name);
3497
3498 info->size = find_data.nFileSizeHigh;
3499 info->size <<= 32;
3500 info->size |= find_data.nFileSizeLow;
3501
3502 if (find_data.dwFileAttributes
3503 & FILE_ATTRIBUTE_REPARSE_POINT) {
3504
3505 /* TODO: test Windows symlinks */
3506 /* TODO: MySQL has apparently its own symlink
3507 implementation in Windows, dbname.sym can
3508 redirect a database directory:
3509 REFMAN "windows-symbolic-links.html" */
3510
3511 info->type = OS_FILE_TYPE_LINK;
3512
3513 } else if (find_data.dwFileAttributes
3514 & FILE_ATTRIBUTE_DIRECTORY) {
3515
3516 info->type = OS_FILE_TYPE_DIR;
3517
3518 } else {
3519
3520 /* It is probably safest to assume that all other
3521 file types are normal. Better to check them rather
3522 than blindly skip them. */
3523
3524 info->type = OS_FILE_TYPE_FILE;
3525 }
3526
3527 status = 0;
3528
3529 } else {
3530 DWORD err = GetLastError();
3531 if (err == ERROR_NO_MORE_FILES) {
3532 status = 1;
3533 } else {
3534 msg("FindNextFile in %s returned %lu", dirname, err);
3535 status = -1;
3536 }
3537 }
3538
3539 return(status);
3540 #else
3541 struct dirent* ent;
3542 char* full_path;
3543 int ret;
3544 struct stat statinfo;
3545
3546 next_file:
3547
3548 ent = readdir(dir);
3549
3550 if (ent == NULL) {
3551
3552 return(1);
3553 }
3554
3555 ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH);
3556
3557 if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
3558
3559 goto next_file;
3560 }
3561
3562 strcpy(info->name, ent->d_name);
3563
3564 full_path = static_cast<char*>(
3565 ut_malloc_nokey(strlen(dirname) + strlen(ent->d_name) + 10));
3566
3567 sprintf(full_path, "%s/%s", dirname, ent->d_name);
3568
3569 ret = stat(full_path, &statinfo);
3570
3571 if (ret) {
3572
3573 if (errno == ENOENT) {
3574 /* readdir() returned a file that does not exist,
3575 it must have been deleted in the meantime. Do what
3576 would have happened if the file was deleted before
3577 readdir() - ignore and go to the next entry.
3578 If this is the last entry then info->name will still
3579 contain the name of the deleted file when this
3580 function returns, but this is not an issue since the
3581 caller shouldn't be looking at info when end of
3582 directory is returned. */
3583
3584 ut_free(full_path);
3585
3586 goto next_file;
3587 }
3588
3589 msg("stat %s: Got error %d", full_path, errno);
3590
3591 ut_free(full_path);
3592
3593 return(-1);
3594 }
3595
3596 info->size = statinfo.st_size;
3597
3598 if (S_ISDIR(statinfo.st_mode)) {
3599 info->type = OS_FILE_TYPE_DIR;
3600 } else if (S_ISLNK(statinfo.st_mode)) {
3601 info->type = OS_FILE_TYPE_LINK;
3602 } else if (S_ISREG(statinfo.st_mode)) {
3603 info->type = OS_FILE_TYPE_FILE;
3604 } else {
3605 info->type = OS_FILE_TYPE_UNKNOWN;
3606 }
3607
3608 ut_free(full_path);
3609 return(0);
3610 #endif
3611 }
3612
3613 /***********************************************************************//**
3614 A fault-tolerant function that tries to read the next file name in the
3615 directory. We retry 100 times if os_file_readdir_next_file() returns -1. The
3616 idea is to read as much good data as we can and jump over bad data.
3617 @return 0 if ok, -1 if error even after the retries, 1 if at the end
3618 of the directory */
3619 int
fil_file_readdir_next_file(dberr_t * err,const char * dirname,os_file_dir_t dir,os_file_stat_t * info)3620 fil_file_readdir_next_file(
3621 /*=======================*/
3622 dberr_t* err, /*!< out: this is set to DB_ERROR if an error
3623 was encountered, otherwise not changed */
3624 const char* dirname,/*!< in: directory name or path */
3625 os_file_dir_t dir, /*!< in: directory stream */
3626 os_file_stat_t* info) /*!< in/out: buffer where the
3627 info is returned */
3628 {
3629 for (ulint i = 0; i < 100; i++) {
3630 int ret = os_file_readdir_next_file(dirname, dir, info);
3631
3632 if (ret != -1) {
3633
3634 return(ret);
3635 }
3636
3637 ib::error() << "os_file_readdir_next_file() returned -1 in"
3638 " directory " << dirname
3639 << ", crash recovery may have failed"
3640 " for some .ibd files!";
3641
3642 *err = DB_ERROR;
3643 }
3644
3645 return(-1);
3646 }
3647
3648 /** Scan the database directories under the MySQL datadir, looking for
3649 .ibd files and determining the space id in each of them.
3650 @return DB_SUCCESS or error number */
3651
enumerate_ibd_files(process_single_tablespace_func_t callback)3652 static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback)
3653 {
3654 int ret;
3655 char* dbpath = NULL;
3656 ulint dbpath_len = 100;
3657 os_file_dir_t dir;
3658 os_file_dir_t dbdir;
3659 os_file_stat_t dbinfo;
3660 os_file_stat_t fileinfo;
3661 dberr_t err = DB_SUCCESS;
3662 size_t len;
3663
3664 /* The datadir of MySQL is always the default directory of mysqld */
3665
3666 dir = os_file_opendir(fil_path_to_mysql_datadir);
3667
3668 if (UNIV_UNLIKELY(dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr))) {
3669 msg("cannot open dir %s", fil_path_to_mysql_datadir);
3670 return(DB_ERROR);
3671 }
3672
3673 dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
3674
3675 /* Scan all directories under the datadir. They are the database
3676 directories of MySQL. */
3677
3678 ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir,
3679 &dbinfo);
3680 while (ret == 0) {
3681
3682 /* General tablespaces are always at the first level of the
3683 data home dir */
3684 if (dbinfo.type == OS_FILE_TYPE_FILE) {
3685 bool is_isl = ends_with(dbinfo.name, ".isl");
3686 bool is_ibd = !is_isl && ends_with(dbinfo.name,".ibd");
3687
3688 if (is_isl || is_ibd) {
3689 (*callback)(NULL, dbinfo.name, is_isl, false);
3690 }
3691 }
3692
3693 if (dbinfo.type == OS_FILE_TYPE_FILE
3694 || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
3695
3696 goto next_datadir_item;
3697 }
3698
3699 /* We found a symlink or a directory; try opening it to see
3700 if a symlink is a directory */
3701
3702 len = strlen(fil_path_to_mysql_datadir)
3703 + strlen (dbinfo.name) + 2;
3704 if (len > dbpath_len) {
3705 dbpath_len = len;
3706
3707 if (dbpath) {
3708 ut_free(dbpath);
3709 }
3710
3711 dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
3712 }
3713 snprintf(dbpath, dbpath_len,
3714 "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
3715 os_normalize_path(dbpath);
3716
3717 if (check_if_skip_database_by_path(dbpath)) {
3718 fprintf(stderr, "Skipping db: %s\n", dbpath);
3719 goto next_datadir_item;
3720 }
3721
3722 dbdir = os_file_opendir(dbpath);
3723
3724 if (UNIV_UNLIKELY(dbdir != IF_WIN(INVALID_HANDLE_VALUE,NULL))){
3725 /* We found a database directory; loop through it,
3726 looking for possible .ibd files in it */
3727
3728 for (ret = fil_file_readdir_next_file(&err, dbpath,
3729 dbdir,
3730 &fileinfo);
3731 ret == 0;
3732 ret = fil_file_readdir_next_file(&err, dbpath,
3733 dbdir,
3734 &fileinfo)) {
3735 if (fileinfo.type == OS_FILE_TYPE_DIR) {
3736 continue;
3737 }
3738
3739 /* We found a symlink or a file */
3740 if (strlen(fileinfo.name) > 4) {
3741 bool is_isl= false;
3742 if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".isl"))))
3743 (*callback)(dbinfo.name, fileinfo.name, is_isl, false);
3744 }
3745 }
3746
3747 if (os_file_closedir_failed(dbdir)) {
3748 fprintf(stderr, "InnoDB: Warning: could not"
3749 " close database directory %s\n",
3750 dbpath);
3751
3752 err = DB_ERROR;
3753 }
3754
3755 } else {
3756 msg("Can't open dir %s", dbpath);
3757 err = DB_ERROR;
3758 break;
3759
3760 }
3761
3762 next_datadir_item:
3763 ret = fil_file_readdir_next_file(&err,
3764 fil_path_to_mysql_datadir,
3765 dir, &dbinfo);
3766 }
3767
3768 ut_free(dbpath);
3769
3770 if (os_file_closedir_failed(dir)) {
3771 fprintf(stderr,
3772 "InnoDB: Error: could not close MySQL datadir\n");
3773 return(DB_ERROR);
3774 }
3775
3776 return(err);
3777 }
3778
3779 /** Assign srv_undo_space_id_start variable if there are undo tablespace present.
3780 Read the TRX_SYS page from ibdata1 file and get the minimum space id from
3781 the first slot rollback segments of TRX_SYS_PAGE_NO.
3782 @retval DB_ERROR if file open or page read failed.
3783 @retval DB_SUCCESS if srv_undo_space_id assigned successfully. */
xb_assign_undo_space_start()3784 static dberr_t xb_assign_undo_space_start()
3785 {
3786
3787 pfs_os_file_t file;
3788 byte* buf;
3789 byte* page;
3790 bool ret;
3791 dberr_t error = DB_SUCCESS;
3792 ulint space;
3793 int n_retries = 5;
3794 ulint fsp_flags;
3795
3796 if (srv_undo_tablespaces == 0) {
3797 return error;
3798 }
3799
3800 file = os_file_create(0, srv_sys_space.first_datafile()->filepath(),
3801 OS_FILE_OPEN, OS_FILE_NORMAL, OS_DATA_FILE, true, &ret);
3802
3803 if (!ret) {
3804 msg("Error opening %s", srv_sys_space.first_datafile()->filepath());
3805 return DB_ERROR;
3806 }
3807
3808 buf = static_cast<byte*>(ut_malloc_nokey(2U << srv_page_size_shift));
3809 page = static_cast<byte*>(ut_align(buf, srv_page_size));
3810
3811 if (os_file_read(IORequestRead, file, page, 0, srv_page_size)
3812 != DB_SUCCESS) {
3813 msg("Reading first page failed.\n");
3814 error = DB_ERROR;
3815 goto func_exit;
3816 }
3817
3818 fsp_flags = mach_read_from_4(
3819 page + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS);
3820 retry:
3821 if (os_file_read(IORequestRead, file, page,
3822 TRX_SYS_PAGE_NO << srv_page_size_shift,
3823 srv_page_size) != DB_SUCCESS) {
3824 msg("Reading TRX_SYS page failed.");
3825 error = DB_ERROR;
3826 goto func_exit;
3827 }
3828
3829 /* TRX_SYS page can't be compressed or encrypted. */
3830 if (buf_page_is_corrupted(false, page, fsp_flags)) {
3831 if (n_retries--) {
3832 os_thread_sleep(1000);
3833 goto retry;
3834 } else {
3835 msg("mariabackup: TRX_SYS page corrupted.\n");
3836 error = DB_ERROR;
3837 goto func_exit;
3838 }
3839 }
3840
3841 /* 0th slot always points to system tablespace.
3842 1st slot should point to first undotablespace which is minimum. */
3843
3844 ut_ad(mach_read_from_4(TRX_SYS + TRX_SYS_RSEGS
3845 + TRX_SYS_RSEG_SLOT_SIZE
3846 + TRX_SYS_RSEG_PAGE_NO + page)
3847 != FIL_NULL);
3848
3849 space = mach_read_from_4(TRX_SYS + TRX_SYS_RSEGS
3850 + TRX_SYS_RSEG_SLOT_SIZE
3851 + TRX_SYS_RSEG_SPACE + page);
3852
3853 srv_undo_space_id_start = space;
3854
3855 func_exit:
3856 ut_free(buf);
3857 ret = os_file_close(file);
3858 ut_a(ret);
3859
3860 return error;
3861 }
3862
3863 /****************************************************************************
3864 Populates the tablespace memory cache by scanning for and opening data files.
3865 @returns DB_SUCCESS or error code.*/
3866 static
3867 dberr_t
xb_load_tablespaces()3868 xb_load_tablespaces()
3869 {
3870 bool create_new_db;
3871 dberr_t err;
3872 ulint sum_of_new_sizes;
3873 lsn_t flush_lsn;
3874
3875 ut_ad(srv_operation == SRV_OPERATION_BACKUP
3876 || srv_operation == SRV_OPERATION_RESTORE_DELTA);
3877
3878 err = srv_sys_space.check_file_spec(&create_new_db, 0);
3879
3880 /* create_new_db must not be true. */
3881 if (err != DB_SUCCESS || create_new_db) {
3882 msg("Could not find data files at the specified datadir");
3883 return(DB_ERROR);
3884 }
3885
3886 for (int i= 0; i < 10; i++) {
3887 err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes,
3888 &flush_lsn);
3889 if (err == DB_PAGE_CORRUPTED || err == DB_CORRUPTION) {
3890 my_sleep(1000);
3891 }
3892 else
3893 break;
3894 }
3895
3896 if (err != DB_SUCCESS) {
3897 msg("Could not open data files.\n");
3898 return(err);
3899 }
3900
3901 /* Add separate undo tablespaces to fil_system */
3902
3903 err = xb_assign_undo_space_start();
3904
3905 if (err != DB_SUCCESS) {
3906 return err;
3907 }
3908
3909 err = srv_undo_tablespaces_init(false);
3910
3911 if (err != DB_SUCCESS) {
3912 return(err);
3913 }
3914
3915 /* It is important to call xb_load_single_table_tablespaces() after
3916 srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() *
3917 relies on srv_undo_tablespaces_open to be properly initialized */
3918
3919 msg("mariabackup: Generating a list of tablespaces");
3920
3921 err = enumerate_ibd_files(xb_load_single_table_tablespace);
3922 if (err != DB_SUCCESS) {
3923 return(err);
3924 }
3925 DBUG_MARIABACKUP_EVENT("after_load_tablespaces", 0);
3926 return(DB_SUCCESS);
3927 }
3928
3929 /************************************************************************
3930 Initialize the tablespace memory cache and populate it by scanning for and
3931 opening data files.
3932 @returns DB_SUCCESS or error code.*/
3933 static
3934 dberr_t
xb_data_files_init()3935 xb_data_files_init()
3936 {
3937 xb_fil_io_init();
3938
3939 return(xb_load_tablespaces());
3940 }
3941
3942 /************************************************************************
3943 Destroy the tablespace memory cache. */
3944 static
3945 void
xb_data_files_close()3946 xb_data_files_close()
3947 {
3948 ut_ad(!os_thread_count);
3949 fil_close_all_files();
3950 if (buf_dblwr) {
3951 buf_dblwr_free();
3952 }
3953 }
3954
3955 /***********************************************************************
3956 Allocate and initialize the entry for databases and tables filtering
3957 hash tables. If memory allocation is not successful, terminate program.
3958 @return pointer to the created entry. */
3959 static
3960 xb_filter_entry_t *
xb_new_filter_entry(const char * name)3961 xb_new_filter_entry(
3962 /*================*/
3963 const char* name) /*!< in: name of table/database */
3964 {
3965 xb_filter_entry_t *entry;
3966 ulint namelen = strlen(name);
3967
3968 ut_a(namelen <= NAME_LEN * 2 + 1);
3969
3970 entry = static_cast<xb_filter_entry_t *>
3971 (malloc(sizeof(xb_filter_entry_t) + namelen + 1));
3972 memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1);
3973 entry->name = ((char*)entry) + sizeof(xb_filter_entry_t);
3974 strcpy(entry->name, name);
3975 entry->has_tables = FALSE;
3976
3977 return entry;
3978 }
3979
3980 /***********************************************************************
3981 Add entry to hash table. If hash table is NULL, allocate and initialize
3982 new hash table */
3983 static
3984 xb_filter_entry_t*
xb_add_filter(const char * name,hash_table_t ** hash)3985 xb_add_filter(
3986 /*========================*/
3987 const char* name, /*!< in: name of table/database */
3988 hash_table_t** hash) /*!< in/out: hash to insert into */
3989 {
3990 xb_filter_entry_t* entry;
3991
3992 entry = xb_new_filter_entry(name);
3993
3994 if (UNIV_UNLIKELY(*hash == NULL)) {
3995 *hash = hash_create(1000);
3996 }
3997 HASH_INSERT(xb_filter_entry_t,
3998 name_hash, *hash,
3999 ut_fold_string(entry->name),
4000 entry);
4001
4002 return entry;
4003 }
4004
4005 /***********************************************************************
4006 Validate name of table or database. If name is invalid, program will
4007 be finished with error code */
4008 static
4009 void
xb_validate_name(const char * name,size_t len)4010 xb_validate_name(
4011 /*=============*/
4012 const char* name, /*!< in: name */
4013 size_t len) /*!< in: length of name */
4014 {
4015 const char* p;
4016
4017 /* perform only basic validation. validate length and
4018 path symbols */
4019 if (len > NAME_LEN) {
4020 die("name `%s` is too long.", name);
4021 }
4022 p = strpbrk(name, "/\\~");
4023 if (p && (uint) (p - name) < NAME_LEN) {
4024 die("name `%s` is not valid.", name);
4025 }
4026 }
4027
4028 /***********************************************************************
4029 Register new filter entry which can be either database
4030 or table name. */
4031 static
4032 void
xb_register_filter_entry(const char * name,hash_table_t ** databases_hash,hash_table_t ** tables_hash)4033 xb_register_filter_entry(
4034 /*=====================*/
4035 const char* name, /*!< in: name */
4036 hash_table_t** databases_hash,
4037 hash_table_t** tables_hash
4038 )
4039 {
4040 const char* p;
4041 size_t namelen;
4042 xb_filter_entry_t* db_entry = NULL;
4043
4044 namelen = strlen(name);
4045 if ((p = strchr(name, '.')) != NULL) {
4046 char dbname[NAME_LEN + 1];
4047
4048 xb_validate_name(name, p - name);
4049 xb_validate_name(p + 1, namelen - (p - name));
4050
4051 strncpy(dbname, name, p - name);
4052 dbname[p - name] = 0;
4053
4054 if (*databases_hash) {
4055 HASH_SEARCH(name_hash, (*databases_hash),
4056 ut_fold_string(dbname),
4057 xb_filter_entry_t*,
4058 db_entry, (void) 0,
4059 !strcmp(db_entry->name, dbname));
4060 }
4061 if (!db_entry) {
4062 db_entry = xb_add_filter(dbname, databases_hash);
4063 }
4064 db_entry->has_tables = TRUE;
4065 xb_add_filter(name, tables_hash);
4066 } else {
4067 xb_validate_name(name, namelen);
4068
4069 xb_add_filter(name, databases_hash);
4070 }
4071 }
4072
4073 static
4074 void
xb_register_include_filter_entry(const char * name)4075 xb_register_include_filter_entry(
4076 const char* name
4077 )
4078 {
4079 xb_register_filter_entry(name, &databases_include_hash,
4080 &tables_include_hash);
4081 }
4082
4083 static
4084 void
xb_register_exclude_filter_entry(const char * name)4085 xb_register_exclude_filter_entry(
4086 const char* name
4087 )
4088 {
4089 xb_register_filter_entry(name, &databases_exclude_hash,
4090 &tables_exclude_hash);
4091 }
4092
register_ignore_db_dirs_filter(const char * name)4093 void register_ignore_db_dirs_filter(const char *name)
4094 {
4095 xb_add_filter(name, &databases_exclude_hash);
4096 }
4097
4098 /***********************************************************************
4099 Register new table for the filter. */
4100 static
4101 void
xb_register_table(const char * name)4102 xb_register_table(
4103 /*==============*/
4104 const char* name) /*!< in: name of table */
4105 {
4106 if (strchr(name, '.') == NULL) {
4107 die("`%s` is not fully qualified name.", name);
4108 }
4109
4110 xb_register_include_filter_entry(name);
4111 }
4112
4113 static
4114 void
xb_add_regex_to_list(const char * regex,const char * error_context,regex_list_t * list)4115 xb_add_regex_to_list(
4116 const char* regex, /*!< in: regex */
4117 const char* error_context, /*!< in: context to error message */
4118 regex_list_t* list) /*! in: list to put new regex to */
4119 {
4120 char errbuf[100];
4121 int ret;
4122
4123 regex_t compiled_regex;
4124 ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
4125
4126 if (ret != 0) {
4127 regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
4128 msg("mariabackup: error: %s regcomp(%s): %s",
4129 error_context, regex, errbuf);
4130 exit(EXIT_FAILURE);
4131 }
4132
4133 list->push_back(compiled_regex);
4134 }
4135
4136 /***********************************************************************
4137 Register new regex for the include filter. */
4138 static
4139 void
xb_register_include_regex(const char * regex)4140 xb_register_include_regex(
4141 /*==============*/
4142 const char* regex) /*!< in: regex */
4143 {
4144 xb_add_regex_to_list(regex, "tables", ®ex_include_list);
4145 }
4146
4147 /***********************************************************************
4148 Register new regex for the exclude filter. */
4149 static
4150 void
xb_register_exclude_regex(const char * regex)4151 xb_register_exclude_regex(
4152 /*==============*/
4153 const char* regex) /*!< in: regex */
4154 {
4155 xb_add_regex_to_list(regex, "tables-exclude", ®ex_exclude_list);
4156 }
4157
4158 typedef void (*insert_entry_func_t)(const char*);
4159
4160 /* Scan string and load filter entries from it.
4161 @param[in] list string representing a list
4162 @param[in] delimiters delimiters of entries
4163 @param[in] ins callback to add entry */
xb_load_list_string(char * list,const char * delimiters,insert_entry_func_t ins)4164 void xb_load_list_string(char *list, const char *delimiters,
4165 insert_entry_func_t ins)
4166 {
4167 char *p;
4168 char *saveptr;
4169
4170 p= strtok_r(list, delimiters, &saveptr);
4171 while (p)
4172 {
4173
4174 ins(p);
4175
4176 p= strtok_r(NULL, delimiters, &saveptr);
4177 }
4178 }
4179
4180 /***********************************************************************
4181 Scan file and load filter entries from it. */
4182 static
4183 void
xb_load_list_file(const char * filename,insert_entry_func_t ins)4184 xb_load_list_file(
4185 /*==============*/
4186 const char* filename, /*!< in: name of file */
4187 insert_entry_func_t ins) /*!< in: callback to add entry */
4188 {
4189 char name_buf[NAME_LEN*2+2];
4190 FILE* fp;
4191
4192 /* read and store the filenames */
4193 fp = fopen(filename, "r");
4194 if (!fp) {
4195 die("Can't open %s",
4196 filename);
4197 }
4198 while (fgets(name_buf, sizeof(name_buf), fp) != NULL) {
4199 char* p = strchr(name_buf, '\n');
4200 if (p) {
4201 *p = '\0';
4202 } else {
4203 die("`%s...` name is too long", name_buf);
4204 }
4205
4206 ins(name_buf);
4207 }
4208
4209 fclose(fp);
4210 }
4211
4212
4213 static
4214 void
xb_filters_init()4215 xb_filters_init()
4216 {
4217 if (xtrabackup_databases) {
4218 xb_load_list_string(xtrabackup_databases, " \t",
4219 xb_register_include_filter_entry);
4220 }
4221
4222 if (xtrabackup_databases_file) {
4223 xb_load_list_file(xtrabackup_databases_file,
4224 xb_register_include_filter_entry);
4225 }
4226
4227 if (xtrabackup_databases_exclude) {
4228 xb_load_list_string(xtrabackup_databases_exclude, " \t",
4229 xb_register_exclude_filter_entry);
4230 }
4231
4232 if (xtrabackup_tables) {
4233 xb_load_list_string(xtrabackup_tables, ",",
4234 xb_register_include_regex);
4235 }
4236
4237 if (xtrabackup_tables_file) {
4238 xb_load_list_file(xtrabackup_tables_file, xb_register_table);
4239 }
4240
4241 if (xtrabackup_tables_exclude) {
4242 xb_load_list_string(xtrabackup_tables_exclude, ",",
4243 xb_register_exclude_regex);
4244 }
4245 }
4246
4247 static
4248 void
xb_filter_hash_free(hash_table_t * hash)4249 xb_filter_hash_free(hash_table_t* hash)
4250 {
4251 ulint i;
4252
4253 /* free the hash elements */
4254 for (i = 0; i < hash_get_n_cells(hash); i++) {
4255 xb_filter_entry_t* table;
4256
4257 table = static_cast<xb_filter_entry_t *>
4258 (HASH_GET_FIRST(hash, i));
4259
4260 while (table) {
4261 xb_filter_entry_t* prev_table = table;
4262
4263 table = static_cast<xb_filter_entry_t *>
4264 (HASH_GET_NEXT(name_hash, prev_table));
4265
4266 HASH_DELETE(xb_filter_entry_t, name_hash, hash,
4267 ut_fold_string(prev_table->name), prev_table);
4268 free(prev_table);
4269 }
4270 }
4271
4272 /* free hash */
4273 hash_table_free(hash);
4274 }
4275
xb_regex_list_free(regex_list_t * list)4276 static void xb_regex_list_free(regex_list_t* list)
4277 {
4278 while (list->size() > 0) {
4279 xb_regfree(&list->front());
4280 list->pop_front();
4281 }
4282 }
4283
4284 /************************************************************************
4285 Destroy table filters for partial backup. */
4286 static
4287 void
xb_filters_free()4288 xb_filters_free()
4289 {
4290 xb_regex_list_free(®ex_include_list);
4291 xb_regex_list_free(®ex_exclude_list);
4292
4293 if (tables_include_hash) {
4294 xb_filter_hash_free(tables_include_hash);
4295 }
4296
4297 if (tables_exclude_hash) {
4298 xb_filter_hash_free(tables_exclude_hash);
4299 }
4300
4301 if (databases_include_hash) {
4302 xb_filter_hash_free(databases_include_hash);
4303 }
4304
4305 if (databases_exclude_hash) {
4306 xb_filter_hash_free(databases_exclude_hash);
4307 }
4308 }
4309
4310 /*********************************************************************//**
4311 Create log file metadata. */
4312 static
4313 void
open_or_create_log_file(fil_space_t * space,ulint i)4314 open_or_create_log_file(
4315 /*====================*/
4316 fil_space_t* space,
4317 ulint i) /*!< in: log file number in group */
4318 {
4319 char name[FN_REFLEN];
4320 ulint dirnamelen;
4321
4322 os_normalize_path(srv_log_group_home_dir);
4323
4324 dirnamelen = strlen(srv_log_group_home_dir);
4325 ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile");
4326 memcpy(name, srv_log_group_home_dir, dirnamelen);
4327
4328 /* Add a path separator if needed. */
4329 if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) {
4330 name[dirnamelen++] = OS_PATH_SEPARATOR;
4331 }
4332
4333 sprintf(name + dirnamelen, "%s%zu", "ib_logfile", i);
4334
4335 ut_a(fil_validate());
4336
4337 space->add(name, OS_FILE_CLOSED,
4338 ulint(srv_log_file_size >> srv_page_size_shift),
4339 false, false);
4340 }
4341
4342 /***********************************************************************
4343 Set the open files limit. Based on set_max_open_files().
4344
4345 @return the resulting open files limit. May be less or more than the requested
4346 value. */
4347 static uint
xb_set_max_open_files(uint max_file_limit)4348 xb_set_max_open_files(
4349 /*==================*/
4350 uint max_file_limit) /*!<in: open files limit */
4351 {
4352 #if defined(RLIMIT_NOFILE)
4353 struct rlimit rlimit;
4354 uint old_cur;
4355
4356 if (getrlimit(RLIMIT_NOFILE, &rlimit)) {
4357
4358 goto end;
4359 }
4360
4361 old_cur = (uint) rlimit.rlim_cur;
4362
4363 if (rlimit.rlim_cur == RLIM_INFINITY) {
4364
4365 rlimit.rlim_cur = max_file_limit;
4366 }
4367
4368 if (rlimit.rlim_cur >= max_file_limit) {
4369
4370 max_file_limit = rlimit.rlim_cur;
4371 goto end;
4372 }
4373
4374 rlimit.rlim_cur = rlimit.rlim_max = max_file_limit;
4375
4376 if (setrlimit(RLIMIT_NOFILE, &rlimit)) {
4377
4378 max_file_limit = old_cur; /* Use original value */
4379 } else {
4380
4381 rlimit.rlim_cur = 0; /* Safety if next call fails */
4382
4383 (void) getrlimit(RLIMIT_NOFILE, &rlimit);
4384
4385 if (rlimit.rlim_cur) {
4386
4387 /* If call didn't fail */
4388 max_file_limit = (uint) rlimit.rlim_cur;
4389 }
4390 }
4391
4392 end:
4393 return(max_file_limit);
4394 #else
4395 return(0);
4396 #endif
4397 }
4398
stop_backup_threads()4399 static void stop_backup_threads()
4400 {
4401 if (log_copying_stop && log_copying_running) {
4402 os_event_set(log_copying_stop);
4403 fputs("mariabackup: Stopping log copying thread", stderr);
4404 fflush(stderr);
4405 while (log_copying_running) {
4406 putc('.', stderr);
4407 fflush(stderr);
4408 os_thread_sleep(200000); /*0.2 sec*/
4409 }
4410 putc('\n', stderr);
4411 os_event_destroy(log_copying_stop);
4412 }
4413
4414 if (wait_throttle) {
4415 /* wait for io_watching_thread completion */
4416 while (io_watching_thread_running) {
4417 os_thread_sleep(1000000);
4418 }
4419 os_event_destroy(wait_throttle);
4420 }
4421 }
4422
4423 /** Implement the core of --backup
4424 @return whether the operation succeeded */
xtrabackup_backup_low()4425 static bool xtrabackup_backup_low()
4426 {
4427 ut_ad(!metadata_to_lsn);
4428
4429 /* read the latest checkpoint lsn */
4430 {
4431 ulint max_cp_field;
4432
4433 log_mutex_enter();
4434
4435 if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS
4436 && log_sys.log.format != 0) {
4437 if (max_cp_field == LOG_CHECKPOINT_1) {
4438 log_header_read(max_cp_field);
4439 }
4440 metadata_to_lsn = mach_read_from_8(
4441 log_sys.checkpoint_buf + LOG_CHECKPOINT_LSN);
4442 msg("mariabackup: The latest check point"
4443 " (for incremental): '" LSN_PF "'",
4444 metadata_to_lsn);
4445 } else {
4446 msg("Error: recv_find_max_checkpoint() failed.");
4447 }
4448 log_mutex_exit();
4449 }
4450
4451 stop_backup_threads();
4452
4453 if (metadata_to_lsn && xtrabackup_copy_logfile(true)) {
4454 ds_close(dst_log_file);
4455 dst_log_file = NULL;
4456 return false;
4457 }
4458
4459 if (ds_close(dst_log_file) || !metadata_to_lsn) {
4460 dst_log_file = NULL;
4461 return false;
4462 }
4463
4464 dst_log_file = NULL;
4465
4466 if(!xtrabackup_incremental) {
4467 strcpy(metadata_type, "full-backuped");
4468 metadata_from_lsn = 0;
4469 } else {
4470 strcpy(metadata_type, "incremental");
4471 metadata_from_lsn = incremental_lsn;
4472 }
4473 metadata_last_lsn = log_copy_scanned_lsn;
4474
4475 if (!xtrabackup_stream_metadata(ds_meta)) {
4476 msg("Error: failed to stream metadata.");
4477 return false;
4478 }
4479 if (xtrabackup_extra_lsndir) {
4480 char filename[FN_REFLEN];
4481
4482 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
4483 XTRABACKUP_METADATA_FILENAME);
4484 if (!xtrabackup_write_metadata(filename)) {
4485 msg("Error: failed to write metadata "
4486 "to '%s'.", filename);
4487 return false;
4488 }
4489 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
4490 XTRABACKUP_INFO);
4491 if (!write_xtrabackup_info(mysql_connection, filename, false, false)) {
4492 msg("Error: failed to write info "
4493 "to '%s'.", filename);
4494 return false;
4495 }
4496 }
4497
4498 return true;
4499 }
4500
4501 /** Implement --backup
4502 @return whether the operation succeeded */
xtrabackup_backup_func()4503 static bool xtrabackup_backup_func()
4504 {
4505 MY_STAT stat_info;
4506 uint i;
4507 uint count;
4508 pthread_mutex_t count_mutex;
4509 CorruptedPages corrupted_pages;
4510 data_thread_ctxt_t *data_threads;
4511 pthread_mutex_init(&backup_mutex, NULL);
4512 pthread_cond_init(&scanned_lsn_cond, NULL);
4513
4514 #ifdef USE_POSIX_FADVISE
4515 msg("uses posix_fadvise().");
4516 #endif
4517
4518 /* cd to datadir */
4519
4520 if (my_setwd(mysql_real_data_home,MYF(MY_WME)))
4521 {
4522 msg("my_setwd() failed , %s", mysql_real_data_home);
4523 return(false);
4524 }
4525 msg("cd to %s", mysql_real_data_home);
4526 encryption_plugin_backup_init(mysql_connection);
4527 msg("open files limit requested %u, set to %u",
4528 (uint) xb_open_files_limit,
4529 xb_set_max_open_files(xb_open_files_limit));
4530
4531 mysql_data_home= mysql_data_home_buff;
4532 mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
4533 mysql_data_home[1]=0;
4534
4535 srv_n_purge_threads = 1;
4536 srv_read_only_mode = TRUE;
4537
4538 srv_operation = SRV_OPERATION_BACKUP;
4539 log_file_op = backup_file_op;
4540 metadata_to_lsn = 0;
4541
4542 /* initialize components */
4543 if(innodb_init_param()) {
4544 fail:
4545 metadata_to_lsn = log_copying_running;
4546 stop_backup_threads();
4547 log_file_op = NULL;
4548 if (dst_log_file) {
4549 ds_close(dst_log_file);
4550 dst_log_file = NULL;
4551 }
4552 if (fil_system.is_initialised()) {
4553 innodb_shutdown();
4554 }
4555 return(false);
4556 }
4557
4558 if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
4559 /* Here we still have srv_pool_size counted
4560 in kilobytes (in 4.0 this was in bytes)
4561 srv_boot() converts the value to
4562 pages; if buffer pool is less than 1000 MB,
4563 assume fewer threads. */
4564 srv_max_n_threads = 50000;
4565
4566 } else if (srv_buf_pool_size >= 8 * 1024 * 1024) {
4567
4568 srv_max_n_threads = 10000;
4569 } else {
4570 srv_max_n_threads = 1000; /* saves several MB of memory,
4571 especially in 64-bit
4572 computers */
4573 }
4574
4575 sync_check_init();
4576 ut_d(sync_check_enable());
4577 /* Reset the system variables in the recovery module. */
4578 recv_sys_var_init();
4579 trx_pool_init();
4580
4581 ut_crc32_init();
4582 crc_init();
4583 recv_sys.create();
4584
4585 #ifdef WITH_INNODB_DISALLOW_WRITES
4586 srv_allow_writes_event = os_event_create(0);
4587 os_event_set(srv_allow_writes_event);
4588 #endif
4589
4590 xb_filters_init();
4591
4592 xb_fil_io_init();
4593 srv_n_file_io_threads = srv_n_read_io_threads;
4594
4595 os_aio_init(srv_n_read_io_threads, srv_n_write_io_threads,
4596 SRV_MAX_N_PENDING_SYNC_IOS);
4597
4598 log_sys.create();
4599 log_sys.log.create(srv_n_log_files);
4600 fil_space_t* space = fil_space_create(
4601 "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0,
4602 FIL_TYPE_LOG, NULL);
4603
4604 for (ulint i = 0; i < srv_n_log_files; i++) {
4605 open_or_create_log_file(space, i);
4606 }
4607
4608 /* create extra LSN dir if it does not exist. */
4609 if (xtrabackup_extra_lsndir
4610 &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0))
4611 && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) {
4612 msg("Error: cannot mkdir %d: %s\n",
4613 my_errno, xtrabackup_extra_lsndir);
4614 goto fail;
4615 }
4616
4617 /* create target dir if not exist */
4618 if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
4619 && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){
4620 msg("Error: cannot mkdir %d: %s\n",
4621 my_errno, xtrabackup_target_dir);
4622 goto fail;
4623 }
4624
4625 {
4626 /* definition from recv_recovery_from_checkpoint_start() */
4627 ulint max_cp_field;
4628
4629 /* start back ground thread to copy newer log */
4630 os_thread_id_t log_copying_thread_id;
4631
4632 /* get current checkpoint_lsn */
4633 /* Look for the latest checkpoint from any of the log groups */
4634
4635 log_mutex_enter();
4636
4637 reread_log_header:
4638 dberr_t err = recv_find_max_checkpoint(&max_cp_field);
4639
4640 if (err != DB_SUCCESS) {
4641 msg("Error: cannot read redo log header");
4642 log_mutex_exit();
4643 goto fail;
4644 }
4645
4646 if (log_sys.log.format == 0) {
4647 msg("Error: cannot process redo log before MariaDB 10.2.2");
4648 log_mutex_exit();
4649 goto fail;
4650 }
4651
4652 const byte* buf = log_sys.checkpoint_buf;
4653 checkpoint_lsn_start = log_sys.log.get_lsn();
4654 checkpoint_no_start = log_sys.next_checkpoint_no;
4655
4656 log_header_read(max_cp_field);
4657
4658 if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)
4659 || checkpoint_lsn_start
4660 != mach_read_from_8(buf + LOG_CHECKPOINT_LSN)
4661 || log_sys.log.get_lsn_offset()
4662 != mach_read_from_8(buf + LOG_CHECKPOINT_OFFSET))
4663 goto reread_log_header;
4664
4665 log_mutex_exit();
4666
4667 xtrabackup_init_datasinks();
4668
4669 if (!select_history()) {
4670 goto fail;
4671 }
4672
4673 /* open the log file */
4674 memset(&stat_info, 0, sizeof(MY_STAT));
4675 dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info);
4676 if (dst_log_file == NULL) {
4677 msg("Error: failed to open the target stream for "
4678 "'ib_logfile0'.");
4679 goto fail;
4680 }
4681
4682 /* label it */
4683 byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr_buf[LOG_FILE_HDR_SIZE];
4684 memset(log_hdr_buf, 0, sizeof log_hdr_buf);
4685
4686 byte *log_hdr_field = log_hdr_buf;
4687 mach_write_to_4(LOG_HEADER_FORMAT + log_hdr_field, log_sys.log.format);
4688 mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr_field, log_sys.log.subformat);
4689 mach_write_to_8(LOG_HEADER_START_LSN + log_hdr_field, checkpoint_lsn_start);
4690 strcpy(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr_field),
4691 "Backup " MYSQL_SERVER_VERSION);
4692 log_block_set_checksum(log_hdr_field,
4693 log_block_calc_checksum_crc32(log_hdr_field));
4694
4695 /* copied from log_group_checkpoint() */
4696 log_hdr_field +=
4697 (log_sys.next_checkpoint_no & 1) ? LOG_CHECKPOINT_2 : LOG_CHECKPOINT_1;
4698 /* The least significant bits of LOG_CHECKPOINT_OFFSET must be
4699 stored correctly in the copy of the ib_logfile. The most significant
4700 bits, which identify the start offset of the log block in the file,
4701 we did choose freely, as LOG_FILE_HDR_SIZE. */
4702 ut_ad(!((log_sys.log.get_lsn() ^ checkpoint_lsn_start)
4703 & (OS_FILE_LOG_BLOCK_SIZE - 1)));
4704 /* Adjust the checkpoint page. */
4705 memcpy(log_hdr_field, log_sys.checkpoint_buf, OS_FILE_LOG_BLOCK_SIZE);
4706 mach_write_to_8(log_hdr_field + LOG_CHECKPOINT_OFFSET,
4707 (checkpoint_lsn_start & (OS_FILE_LOG_BLOCK_SIZE - 1))
4708 | LOG_FILE_HDR_SIZE);
4709 log_block_set_checksum(log_hdr_field,
4710 log_block_calc_checksum_crc32(log_hdr_field));
4711
4712 /* Write log header*/
4713 if (ds_write(dst_log_file, log_hdr_buf, sizeof(log_hdr_buf))) {
4714 msg("error: write to logfile failed");
4715 goto fail;
4716 }
4717
4718 log_copying_running = true;
4719 /* start io throttle */
4720 if(xtrabackup_throttle) {
4721 os_thread_id_t io_watching_thread_id;
4722
4723 io_ticket = xtrabackup_throttle;
4724 wait_throttle = os_event_create(0);
4725 io_watching_thread_running = true;
4726
4727 os_thread_create(io_watching_thread, NULL,
4728 &io_watching_thread_id);
4729 }
4730
4731 /* Populate fil_system with tablespaces to copy */
4732 err = xb_load_tablespaces();
4733 if (err != DB_SUCCESS) {
4734 msg("merror: xb_load_tablespaces() failed with"
4735 " error %s.", ut_strerr(err));
4736 fail_before_log_copying_thread_start:
4737 log_copying_running = false;
4738 goto fail;
4739 }
4740
4741 /* copy log file by current position */
4742 log_copy_scanned_lsn = checkpoint_lsn_start;
4743 recv_sys.recovered_lsn = log_copy_scanned_lsn;
4744 log_optimized_ddl_op = backup_optimized_ddl_op;
4745
4746 if (xtrabackup_copy_logfile())
4747 goto fail_before_log_copying_thread_start;
4748
4749 DBUG_MARIABACKUP_EVENT("before_innodb_log_copy_thread_started",0);
4750
4751 log_copying_stop = os_event_create(0);
4752 os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
4753
4754 /* FLUSH CHANGED_PAGE_BITMAPS call */
4755 if (!flush_changed_page_bitmaps()) {
4756 goto fail;
4757 }
4758
4759 ut_a(xtrabackup_parallel > 0);
4760
4761 if (xtrabackup_parallel > 1) {
4762 msg("mariabackup: Starting %u threads for parallel data "
4763 "files transfer", xtrabackup_parallel);
4764 }
4765
4766 if (opt_lock_ddl_per_table) {
4767 mdl_lock_all();
4768
4769 DBUG_EXECUTE_IF("check_mdl_lock_works",
4770 dbug_alter_thread_done =
4771 dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
4772 "Waiting for table metadata lock", 0, 0););
4773 }
4774
4775 datafiles_iter_t *it = datafiles_iter_new();
4776 if (it == NULL) {
4777 msg("mariabackup: Error: datafiles_iter_new() failed.");
4778 goto fail;
4779 }
4780
4781 /* Create data copying threads */
4782 data_threads = (data_thread_ctxt_t *)
4783 malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel);
4784 count = xtrabackup_parallel;
4785 pthread_mutex_init(&count_mutex, NULL);
4786
4787 for (i = 0; i < (uint) xtrabackup_parallel; i++) {
4788 data_threads[i].it = it;
4789 data_threads[i].num = i+1;
4790 data_threads[i].count = &count;
4791 data_threads[i].count_mutex = &count_mutex;
4792 data_threads[i].corrupted_pages = &corrupted_pages;
4793 os_thread_create(data_copy_thread_func, data_threads + i,
4794 &data_threads[i].id);
4795 }
4796
4797 /* Wait for threads to exit */
4798 while (1) {
4799 os_thread_sleep(1000000);
4800 pthread_mutex_lock(&count_mutex);
4801 bool stop = count == 0;
4802 pthread_mutex_unlock(&count_mutex);
4803 if (stop) {
4804 break;
4805 }
4806 }
4807
4808 pthread_mutex_destroy(&count_mutex);
4809 free(data_threads);
4810 datafiles_iter_free(it);
4811 }
4812
4813 bool ok = backup_start(corrupted_pages);
4814
4815 if (ok) {
4816 ok = xtrabackup_backup_low();
4817
4818 backup_release();
4819
4820 DBUG_EXECUTE_IF("check_mdl_lock_works",
4821 os_event_wait(dbug_alter_thread_done);
4822 os_event_destroy(dbug_alter_thread_done);
4823 );
4824
4825 if (ok) {
4826 backup_finish();
4827 }
4828 }
4829
4830 if (opt_log_innodb_page_corruption)
4831 ok = corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE);
4832
4833 if (!ok) {
4834 goto fail;
4835 }
4836
4837 if (changed_page_bitmap) {
4838 xb_page_bitmap_deinit(changed_page_bitmap);
4839 }
4840 xtrabackup_destroy_datasinks();
4841
4842 msg("Redo log (from LSN " LSN_PF " to " LSN_PF
4843 ") was copied.", checkpoint_lsn_start, log_copy_scanned_lsn);
4844 xb_filters_free();
4845
4846 xb_data_files_close();
4847
4848 /* Make sure that the latest checkpoint was included */
4849 if (metadata_to_lsn > log_copy_scanned_lsn) {
4850 msg("Error: failed to copy enough redo log ("
4851 "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").",
4852 log_copy_scanned_lsn, metadata_to_lsn);
4853 goto fail;
4854 }
4855
4856 innodb_shutdown();
4857 log_file_op = NULL;
4858 pthread_mutex_destroy(&backup_mutex);
4859 pthread_cond_destroy(&scanned_lsn_cond);
4860 if (!corrupted_pages.empty()) {
4861 ut_ad(opt_log_innodb_page_corruption);
4862 msg("Error: corrupted innodb pages are found and logged to "
4863 MB_CORRUPTED_PAGES_FILE " file");
4864 }
4865 return(true);
4866 }
4867
4868
4869 /**
4870 This function handles DDL changes at the end of backup, under protection of
4871 FTWRL. This ensures consistent backup in presence of DDL.
4872
4873 - New tables, that were created during backup, are now copied into backup.
4874 Also, tablespaces with optimized (no redo loggin DDL) are re-copied into
4875 backup. This tablespaces will get the extension ".new" in the backup
4876
4877 - Tables that were renamed during backup, are marked as renamed
4878 For these, file <old_name>.ren will be created.
4879 The content of the file is the new tablespace name.
4880
4881 - Tables that were deleted during backup, are marked as deleted
4882 For these , an empty file <name>.del will be created
4883
4884 It is the responsibility of the prepare phase to deal with .new, .ren, and .del
4885 files.
4886 */
backup_fix_ddl(CorruptedPages & corrupted_pages)4887 void backup_fix_ddl(CorruptedPages &corrupted_pages)
4888 {
4889 std::set<std::string> new_tables;
4890 std::set<std::string> dropped_tables;
4891 std::map<std::string, std::string> renamed_tables;
4892
4893 /* Disable further DDL on backed up tables (only needed for --no-lock).*/
4894 pthread_mutex_lock(&backup_mutex);
4895 log_file_op = backup_file_op_fail;
4896 log_optimized_ddl_op = backup_optimized_ddl_op_fail;
4897 pthread_mutex_unlock(&backup_mutex);
4898
4899 DBUG_MARIABACKUP_EVENT("backup_fix_ddl",0);
4900
4901 for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin();
4902 iter != ddl_tracker.tables_in_backup.end();
4903 iter++) {
4904
4905 const std::string name = iter->second;
4906 ulint id = iter->first;
4907
4908 if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) {
4909 dropped_tables.insert(name);
4910 corrupted_pages.drop_space(id);
4911 continue;
4912 }
4913
4914 bool has_optimized_ddl =
4915 ddl_tracker.optimized_ddl.find(id) != ddl_tracker.optimized_ddl.end();
4916
4917 if (ddl_tracker.id_to_name.find(id) == ddl_tracker.id_to_name.end()) {
4918 if (has_optimized_ddl) {
4919 new_tables.insert(name);
4920 }
4921 continue;
4922 }
4923
4924 /* tablespace was affected by DDL. */
4925 const std::string new_name = ddl_tracker.id_to_name[id];
4926 if (new_name != name) {
4927 if (has_optimized_ddl) {
4928 /* table was renamed, but we need a full copy
4929 of it because of optimized DDL. We emulate a drop/create.*/
4930 dropped_tables.insert(name);
4931 if (opt_log_innodb_page_corruption)
4932 corrupted_pages.drop_space(id);
4933 new_tables.insert(new_name);
4934 } else {
4935 /* Renamed, and no optimized DDL*/
4936 renamed_tables[name] = new_name;
4937 if (opt_log_innodb_page_corruption)
4938 corrupted_pages.rename_space(id, new_name);
4939 }
4940 } else if (has_optimized_ddl) {
4941 /* Table was recreated, or optimized DDL ran.
4942 In both cases we need a full copy in the backup.*/
4943 new_tables.insert(name);
4944 if (opt_log_innodb_page_corruption)
4945 corrupted_pages.drop_space(id);
4946 }
4947 }
4948
4949 /* Find tables that were created during backup (and not removed).*/
4950 for(space_id_to_name_t::iterator iter = ddl_tracker.id_to_name.begin();
4951 iter != ddl_tracker.id_to_name.end();
4952 iter++) {
4953
4954 ulint id = iter->first;
4955 std::string name = iter->second;
4956
4957 if (ddl_tracker.tables_in_backup.find(id) != ddl_tracker.tables_in_backup.end()) {
4958 /* already processed above */
4959 continue;
4960 }
4961
4962 if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end()) {
4963 dropped_tables.erase(name);
4964 new_tables.insert(name);
4965 if (opt_log_innodb_page_corruption)
4966 corrupted_pages.drop_space(id);
4967 }
4968 }
4969
4970 // Mark tablespaces for rename
4971 for (std::map<std::string, std::string>::iterator iter = renamed_tables.begin();
4972 iter != renamed_tables.end(); ++iter) {
4973 const std::string old_name = iter->first;
4974 std::string new_name = iter->second;
4975 backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str());
4976 }
4977
4978 // Mark tablespaces for drop
4979 for (std::set<std::string>::iterator iter = dropped_tables.begin();
4980 iter != dropped_tables.end();
4981 iter++) {
4982 const std::string name(*iter);
4983 backup_file_printf((name + ".del").c_str(), "%s", "");
4984 }
4985
4986 // Load and copy new tables.
4987 // Close all datanodes first, reload only new tables.
4988 std::vector<fil_node_t *> all_nodes;
4989 datafiles_iter_t *it = datafiles_iter_new();
4990 if (!it)
4991 return;
4992 while (fil_node_t *node = datafiles_iter_next(it)) {
4993 all_nodes.push_back(node);
4994 }
4995 for (size_t i = 0; i < all_nodes.size(); i++) {
4996 fil_node_t *n = all_nodes[i];
4997 if (n->space->id == 0)
4998 continue;
4999 if (n->is_open()) {
5000 mutex_enter(&fil_system.mutex);
5001 n->close();
5002 mutex_exit(&fil_system.mutex);
5003 }
5004 fil_space_free(n->space->id, false);
5005 }
5006 datafiles_iter_free(it);
5007
5008 DBUG_EXECUTE_IF("check_mdl_lock_works", DBUG_ASSERT(new_tables.size() == 0););
5009 for (std::set<std::string>::iterator iter = new_tables.begin();
5010 iter != new_tables.end(); iter++) {
5011 const char *space_name = iter->c_str();
5012 if (check_if_skip_table(space_name))
5013 continue;
5014 xb_load_single_table_tablespace(*iter, false);
5015 }
5016
5017 it = datafiles_iter_new();
5018 if (!it)
5019 return;
5020
5021 while (fil_node_t *node = datafiles_iter_next(it)) {
5022 fil_space_t * space = node->space;
5023 if (!fil_is_user_tablespace_id(space->id))
5024 continue;
5025 std::string dest_name(node->space->name);
5026 dest_name.append(".new");
5027 xtrabackup_copy_datafile(node, 0, dest_name.c_str(), wf_write_through,
5028 corrupted_pages);
5029 }
5030
5031 datafiles_iter_free(it);
5032 }
5033
5034 /* ================= prepare ================= */
5035
5036 /***********************************************************************
5037 Generates path to the meta file path from a given path to an incremental .delta
5038 by replacing trailing ".delta" with ".meta", or returns error if 'delta_path'
5039 does not end with the ".delta" character sequence.
5040 @return TRUE on success, FALSE on error. */
5041 static
5042 ibool
get_meta_path(const char * delta_path,char * meta_path)5043 get_meta_path(
5044 const char *delta_path, /* in: path to a .delta file */
5045 char *meta_path) /* out: path to the corresponding .meta
5046 file */
5047 {
5048 size_t len = strlen(delta_path);
5049
5050 if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) {
5051 return FALSE;
5052 }
5053 memcpy(meta_path, delta_path, len - 6);
5054 strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX);
5055
5056 return TRUE;
5057 }
5058
5059 /****************************************************************//**
5060 Create a new tablespace on disk and return the handle to its opened
5061 file. Code adopted from fil_create_new_single_table_tablespace with
5062 the main difference that only disk file is created without updating
5063 the InnoDB in-memory dictionary data structures.
5064
5065 @return true on success, false on error. */
5066 static
5067 bool
xb_space_create_file(const char * path,ulint space_id,ulint flags,pfs_os_file_t * file)5068 xb_space_create_file(
5069 /*==================*/
5070 const char* path, /*!<in: path to tablespace */
5071 ulint space_id, /*!<in: space id */
5072 ulint flags, /*!<in: tablespace flags */
5073 pfs_os_file_t* file) /*!<out: file handle */
5074 {
5075 bool ret;
5076 byte* buf;
5077 byte* page;
5078
5079 *file = os_file_create_simple_no_error_handling(
5080 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret);
5081 if (!ret) {
5082 msg("Can't create file %s", path);
5083 return ret;
5084 }
5085
5086 ret = os_file_set_size(path, *file,
5087 FIL_IBD_FILE_INITIAL_SIZE
5088 << srv_page_size_shift);
5089 if (!ret) {
5090 msg("mariabackup: cannot set size for file %s", path);
5091 os_file_close(*file);
5092 os_file_delete(0, path);
5093 return ret;
5094 }
5095
5096 buf = static_cast<byte *>(malloc(3U << srv_page_size_shift));
5097 /* Align the memory for file i/o if we might have O_DIRECT set */
5098 page = static_cast<byte *>(ut_align(buf, srv_page_size));
5099
5100 memset(page, '\0', srv_page_size);
5101
5102 fsp_header_init_fields(page, space_id, flags);
5103 mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id);
5104
5105 const ulint zip_size = fil_space_t::zip_size(flags);
5106
5107 if (!zip_size) {
5108 buf_flush_init_for_writing(
5109 NULL, page, NULL, 0,
5110 fil_space_t::full_crc32(flags));
5111
5112 ret = os_file_write(IORequestWrite, path, *file, page, 0,
5113 srv_page_size);
5114 } else {
5115 page_zip_des_t page_zip;
5116 page_zip_set_size(&page_zip, zip_size);
5117 page_zip.data = page + srv_page_size;
5118 fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size);
5119
5120 #ifdef UNIV_DEBUG
5121 page_zip.m_start =
5122 #endif /* UNIV_DEBUG */
5123 page_zip.m_end = page_zip.m_nonempty =
5124 page_zip.n_blobs = 0;
5125
5126 buf_flush_init_for_writing(NULL, page, &page_zip, 0, false);
5127
5128 ret = os_file_write(IORequestWrite, path, *file,
5129 page_zip.data, 0, zip_size);
5130 }
5131
5132 free(buf);
5133
5134 if (ret != DB_SUCCESS) {
5135 msg("mariabackup: could not write the first page to %s",
5136 path);
5137 os_file_close(*file);
5138 os_file_delete(0, path);
5139 return ret;
5140 }
5141
5142 return TRUE;
5143 }
5144
fil_space_get_by_name(const char * name)5145 static fil_space_t* fil_space_get_by_name(const char* name)
5146 {
5147 ut_ad(mutex_own(&fil_system.mutex));
5148 for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system.space_list);
5149 space != NULL;
5150 space = UT_LIST_GET_NEXT(space_list, space))
5151 if (!strcmp(space->name, name)) return space;
5152 return NULL;
5153 }
5154
5155 /***********************************************************************
5156 Searches for matching tablespace file for given .delta file and space_id
5157 in given directory. When matching tablespace found, renames it to match the
5158 name of .delta file. If there was a tablespace with matching name and
5159 mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no
5160 matching file, creates a new tablespace.
5161 @return file handle of matched or created file */
5162 static
5163 pfs_os_file_t
xb_delta_open_matching_space(const char * dbname,const char * name,const xb_delta_info_t & info,char * real_name,size_t real_name_len,bool * success)5164 xb_delta_open_matching_space(
5165 const char* dbname, /* in: path to destination database dir */
5166 const char* name, /* in: name of delta file (without .delta) */
5167 const xb_delta_info_t& info,
5168 char* real_name, /* out: full path of destination file */
5169 size_t real_name_len, /* out: buffer size for real_name */
5170 bool* success) /* out: indicates error. true = success */
5171 {
5172 char dest_dir[FN_REFLEN];
5173 char dest_space_name[FN_REFLEN];
5174 fil_space_t* fil_space;
5175 pfs_os_file_t file;
5176 xb_filter_entry_t* table;
5177
5178 ut_a(dbname != NULL ||
5179 !fil_is_user_tablespace_id(info.space_id) ||
5180 info.space_id == ULINT_UNDEFINED);
5181
5182 *success = false;
5183
5184 if (dbname) {
5185 snprintf(dest_dir, FN_REFLEN, "%s/%s",
5186 xtrabackup_target_dir, dbname);
5187 os_normalize_path(dest_dir);
5188
5189 snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name);
5190 } else {
5191 snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir);
5192 os_normalize_path(dest_dir);
5193
5194 snprintf(dest_space_name, FN_REFLEN, "%s", name);
5195 }
5196
5197 snprintf(real_name, real_name_len,
5198 "%s/%s",
5199 xtrabackup_target_dir, dest_space_name);
5200 os_normalize_path(real_name);
5201 /* Truncate ".ibd" */
5202 dest_space_name[strlen(dest_space_name) - 4] = '\0';
5203
5204 /* Create the database directory if it doesn't exist yet */
5205 if (!os_file_create_directory(dest_dir, FALSE)) {
5206 msg("mariabackup: error: cannot create dir %s", dest_dir);
5207 return file;
5208 }
5209
5210 log_mutex_enter();
5211 if (!fil_is_user_tablespace_id(info.space_id)) {
5212 found:
5213 /* open the file and return its handle */
5214
5215 file = os_file_create_simple_no_error_handling(
5216 0, real_name,
5217 OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success);
5218
5219 if (!*success) {
5220 msg("mariabackup: Cannot open file %s\n", real_name);
5221 }
5222 exit:
5223 log_mutex_exit();
5224 return file;
5225 }
5226
5227 /* remember space name for further reference */
5228 table = static_cast<xb_filter_entry_t *>
5229 (malloc(sizeof(xb_filter_entry_t) +
5230 strlen(dest_space_name) + 1));
5231
5232 table->name = ((char*)table) + sizeof(xb_filter_entry_t);
5233 strcpy(table->name, dest_space_name);
5234 HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash,
5235 ut_fold_string(table->name), table);
5236
5237 mutex_enter(&fil_system.mutex);
5238 fil_space = fil_space_get_by_name(dest_space_name);
5239 mutex_exit(&fil_system.mutex);
5240
5241 if (fil_space != NULL) {
5242 if (fil_space->id == info.space_id
5243 || info.space_id == ULINT_UNDEFINED) {
5244 /* we found matching space */
5245 goto found;
5246 } else {
5247
5248 char tmpname[FN_REFLEN];
5249
5250 snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#" ULINTPF,
5251 dbname, fil_space->id);
5252
5253 msg("mariabackup: Renaming %s to %s.ibd",
5254 fil_space->name, tmpname);
5255
5256 if (fil_space->rename(tmpname, NULL, false)
5257 != DB_SUCCESS) {
5258 msg("mariabackup: Cannot rename %s to %s",
5259 fil_space->name, tmpname);
5260 goto exit;
5261 }
5262 }
5263 }
5264
5265 if (info.space_id == ULINT_UNDEFINED)
5266 {
5267 die("Can't handle DDL operation on tablespace "
5268 "%s\n", dest_space_name);
5269 }
5270 mutex_enter(&fil_system.mutex);
5271 fil_space = fil_space_get_by_id(info.space_id);
5272 mutex_exit(&fil_system.mutex);
5273 if (fil_space != NULL) {
5274 char tmpname[FN_REFLEN];
5275
5276 strncpy(tmpname, dest_space_name, FN_REFLEN);
5277
5278 msg("mariabackup: Renaming %s to %s",
5279 fil_space->name, dest_space_name);
5280
5281 if (fil_space->rename(tmpname, NULL, false) != DB_SUCCESS)
5282 {
5283 msg("mariabackup: Cannot rename %s to %s",
5284 fil_space->name, dest_space_name);
5285 goto exit;
5286 }
5287
5288 goto found;
5289 }
5290
5291 /* No matching space found. create the new one. */
5292 const ulint flags = info.zip_size
5293 ? get_bit_shift(info.page_size
5294 >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
5295 << FSP_FLAGS_POS_ZIP_SSIZE
5296 | FSP_FLAGS_MASK_POST_ANTELOPE
5297 | FSP_FLAGS_MASK_ATOMIC_BLOBS
5298 | (srv_page_size == UNIV_PAGE_SIZE_ORIG
5299 ? 0
5300 : get_bit_shift(srv_page_size
5301 >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
5302 << FSP_FLAGS_POS_PAGE_SSIZE)
5303 : FSP_FLAGS_PAGE_SSIZE();
5304 ut_ad(fil_space_t::zip_size(flags) == info.zip_size);
5305 ut_ad(fil_space_t::physical_size(flags) == info.page_size);
5306
5307 if (fil_space_create(dest_space_name, info.space_id, flags,
5308 FIL_TYPE_TABLESPACE, 0)) {
5309 *success = xb_space_create_file(real_name, info.space_id,
5310 flags, &file);
5311 } else {
5312 msg("Can't create tablespace %s\n", dest_space_name);
5313 }
5314
5315 goto exit;
5316 }
5317
5318 /************************************************************************
5319 Applies a given .delta file to the corresponding data file.
5320 @return TRUE on success */
5321 static
5322 ibool
xtrabackup_apply_delta(const char * dirname,const char * dbname,const char * filename,void *)5323 xtrabackup_apply_delta(
5324 const char* dirname, /* in: dir name of incremental */
5325 const char* dbname, /* in: database name (ibdata: NULL) */
5326 const char* filename, /* in: file name (not a path),
5327 including the .delta extension */
5328 void* /*data*/)
5329 {
5330 pfs_os_file_t src_file;
5331 pfs_os_file_t dst_file;
5332 char src_path[FN_REFLEN];
5333 char dst_path[FN_REFLEN];
5334 char meta_path[FN_REFLEN];
5335 char space_name[FN_REFLEN];
5336 bool success;
5337
5338 ibool last_buffer = FALSE;
5339 ulint page_in_buffer;
5340 ulint incremental_buffers = 0;
5341
5342 xb_delta_info_t info(srv_page_size, 0, SRV_TMP_SPACE_ID);
5343 ulint page_size;
5344 ulint page_size_shift;
5345 byte* incremental_buffer_base = NULL;
5346 byte* incremental_buffer;
5347
5348 size_t offset;
5349
5350 ut_a(xtrabackup_incremental);
5351
5352 if (dbname) {
5353 snprintf(src_path, sizeof(src_path), "%s/%s/%s",
5354 dirname, dbname, filename);
5355 snprintf(dst_path, sizeof(dst_path), "%s/%s/%s",
5356 xtrabackup_real_target_dir, dbname, filename);
5357 } else {
5358 snprintf(src_path, sizeof(src_path), "%s/%s",
5359 dirname, filename);
5360 snprintf(dst_path, sizeof(dst_path), "%s/%s",
5361 xtrabackup_real_target_dir, filename);
5362 }
5363 dst_path[strlen(dst_path) - 6] = '\0';
5364
5365 strncpy(space_name, filename, FN_REFLEN - 1);
5366 space_name[FN_REFLEN - 1] = '\0';
5367 space_name[strlen(space_name) - 6] = 0;
5368
5369 if (!get_meta_path(src_path, meta_path)) {
5370 goto error;
5371 }
5372
5373 os_normalize_path(dst_path);
5374 os_normalize_path(src_path);
5375 os_normalize_path(meta_path);
5376
5377 if (!xb_read_delta_metadata(meta_path, &info)) {
5378 goto error;
5379 }
5380
5381 page_size = info.page_size;
5382 page_size_shift = get_bit_shift(page_size);
5383 msg("page size for %s is %zu bytes",
5384 src_path, page_size);
5385 if (page_size_shift < 10 ||
5386 page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) {
5387 msg("error: invalid value of page_size "
5388 "(%zu bytes) read from %s", page_size, meta_path);
5389 goto error;
5390 }
5391
5392 src_file = os_file_create_simple_no_error_handling(
5393 0, src_path,
5394 OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success);
5395 if (!success) {
5396 os_file_get_last_error(TRUE);
5397 msg("error: can't open %s", src_path);
5398 goto error;
5399 }
5400
5401 posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
5402
5403 dst_file = xb_delta_open_matching_space(
5404 dbname, space_name, info,
5405 dst_path, sizeof(dst_path), &success);
5406 if (!success) {
5407 msg("error: can't open %s", dst_path);
5408 goto error;
5409 }
5410
5411 posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
5412
5413 /* allocate buffer for incremental backup (4096 pages) */
5414 incremental_buffer_base = static_cast<byte *>
5415 (malloc((page_size / 4 + 1) * page_size));
5416 incremental_buffer = static_cast<byte *>
5417 (ut_align(incremental_buffer_base,
5418 page_size));
5419
5420 msg("Applying %s to %s...", src_path, dst_path);
5421
5422 while (!last_buffer) {
5423 ulint cluster_header;
5424
5425 /* read to buffer */
5426 /* first block of block cluster */
5427 offset = ((incremental_buffers * (page_size / 4))
5428 << page_size_shift);
5429 if (os_file_read(IORequestRead, src_file,
5430 incremental_buffer, offset, page_size)
5431 != DB_SUCCESS) {
5432 goto error;
5433 }
5434
5435 cluster_header = mach_read_from_4(incremental_buffer);
5436 switch(cluster_header) {
5437 case 0x78747261UL: /*"xtra"*/
5438 break;
5439 case 0x58545241UL: /*"XTRA"*/
5440 last_buffer = TRUE;
5441 break;
5442 default:
5443 msg("error: %s seems not "
5444 ".delta file.", src_path);
5445 goto error;
5446 }
5447
5448 /* FIXME: If the .delta modifies FSP_SIZE on page 0,
5449 extend the file to that size. */
5450
5451 for (page_in_buffer = 1; page_in_buffer < page_size / 4;
5452 page_in_buffer++) {
5453 if (mach_read_from_4(incremental_buffer + page_in_buffer * 4)
5454 == 0xFFFFFFFFUL)
5455 break;
5456 }
5457
5458 ut_a(last_buffer || page_in_buffer == page_size / 4);
5459
5460 /* read whole of the cluster */
5461 if (os_file_read(IORequestRead, src_file,
5462 incremental_buffer,
5463 offset, page_in_buffer * page_size)
5464 != DB_SUCCESS) {
5465 goto error;
5466 }
5467
5468 posix_fadvise(src_file, offset, page_in_buffer * page_size,
5469 POSIX_FADV_DONTNEED);
5470
5471 for (page_in_buffer = 1; page_in_buffer < page_size / 4;
5472 page_in_buffer++) {
5473 ulint offset_on_page;
5474
5475 offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4);
5476
5477 if (offset_on_page == 0xFFFFFFFFUL)
5478 break;
5479
5480 uchar *buf = incremental_buffer + page_in_buffer * page_size;
5481 const os_offset_t off = os_offset_t(offset_on_page)*page_size;
5482
5483 if (off == 0) {
5484 /* Read tablespace size from page 0,
5485 and extend the file to specified size.*/
5486 os_offset_t n_pages = mach_read_from_4(
5487 buf + FSP_HEADER_OFFSET + FSP_SIZE);
5488 if (mach_read_from_4(buf
5489 + FIL_PAGE_SPACE_ID)) {
5490 if (!os_file_set_size(
5491 dst_path, dst_file,
5492 n_pages * page_size))
5493 goto error;
5494 } else if (fil_space_t* space
5495 = fil_system.sys_space) {
5496 /* The system tablespace can
5497 consist of multiple files. The
5498 first one has full tablespace
5499 size in page 0, but only the last
5500 file should be extended. */
5501 fil_node_t* n = UT_LIST_GET_FIRST(
5502 space->chain);
5503 bool fail = !strcmp(n->name, dst_path)
5504 && !fil_space_extend(
5505 space, (ulint)n_pages);
5506 if (fail) goto error;
5507 }
5508 }
5509
5510 if (os_file_write(IORequestWrite,
5511 dst_path, dst_file, buf, off,
5512 page_size) != DB_SUCCESS) {
5513 goto error;
5514 }
5515 }
5516
5517 /* Free file system buffer cache after the batch was written. */
5518 #ifdef __linux__
5519 os_file_flush_func(dst_file);
5520 #endif
5521 posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
5522
5523
5524 incremental_buffers++;
5525 }
5526
5527 free(incremental_buffer_base);
5528 if (src_file != OS_FILE_CLOSED) {
5529 os_file_close(src_file);
5530 os_file_delete(0,src_path);
5531 }
5532 if (dst_file != OS_FILE_CLOSED)
5533 os_file_close(dst_file);
5534 return TRUE;
5535
5536 error:
5537 free(incremental_buffer_base);
5538 if (src_file != OS_FILE_CLOSED)
5539 os_file_close(src_file);
5540 if (dst_file != OS_FILE_CLOSED)
5541 os_file_close(dst_file);
5542 msg("Error: xtrabackup_apply_delta(): "
5543 "failed to apply %s to %s.\n", src_path, dst_path);
5544 return FALSE;
5545 }
5546
5547
change_extension(std::string filename,std::string new_ext)5548 std::string change_extension(std::string filename, std::string new_ext) {
5549 DBUG_ASSERT(new_ext.size() == 3);
5550 std::string new_name(filename);
5551 new_name.resize(new_name.size() - new_ext.size());
5552 new_name.append(new_ext);
5553 return new_name;
5554 }
5555
5556
rename_file(const char * from,const char * to)5557 static void rename_file(const char *from,const char *to) {
5558 msg("Renaming %s to %s\n", from, to);
5559 if (my_rename(from, to, MY_WME)) {
5560 die("Can't rename %s to %s errno %d", from, to, errno);
5561 }
5562 }
5563
rename_file(const std::string & from,const std::string & to)5564 static void rename_file(const std::string& from, const std::string &to) {
5565 rename_file(from.c_str(), to.c_str());
5566 }
5567 /************************************************************************
5568 Callback to handle datadir entry. Function of this type will be called
5569 for each entry which matches the mask by xb_process_datadir.
5570 @return should return TRUE on success */
5571 typedef ibool (*handle_datadir_entry_func_t)(
5572 /*=========================================*/
5573 const char* data_home_dir, /*!<in: path to datadir */
5574 const char* db_name, /*!<in: database name */
5575 const char* file_name, /*!<in: file name with suffix */
5576 void* arg); /*!<in: caller-provided data */
5577
5578 /** Rename, and replace destination file, if exists */
rename_force(const char * from,const char * to)5579 static void rename_force(const char *from, const char *to) {
5580 if (access(to, R_OK) == 0) {
5581 msg("Removing %s", to);
5582 if (my_delete(to, MYF(MY_WME))) {
5583 msg("Can't remove %s, errno %d", to, errno);
5584 exit(EXIT_FAILURE);
5585 }
5586 }
5587 rename_file(from,to);
5588 }
5589
5590
5591 /** During prepare phase, rename ".new" files, that were created in
5592 backup_fix_ddl() and backup_optimized_ddl_op(), to ".ibd". In the case of
5593 incremental backup, i.e. of arg argument is set, move ".new" files to
5594 destination directory and rename them to ".ibd", remove existing ".ibd.delta"
5595 and ".idb.meta" files in incremental directory to avoid applying delta to
5596 ".ibd" file.
5597
5598 @param[in] data_home_dir path to datadir
5599 @param[in] db_name database name
5600 @param[in] file_name file name with suffix
5601 @param[in] arg destination path, used in incremental backup to notify, that
5602 *.new file must be moved to destibation directory
5603
5604 @return true */
prepare_handle_new_files(const char * data_home_dir,const char * db_name,const char * file_name,void * arg)5605 static ibool prepare_handle_new_files(const char *data_home_dir,
5606 const char *db_name,
5607 const char *file_name, void *arg)
5608 {
5609 const char *dest_dir = static_cast<const char *>(arg);
5610 std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/' + file_name;
5611 /* Copy "*.new" files from incremental to base dir for incremental backup */
5612 std::string dest_path=
5613 dest_dir ? std::string(dest_dir) + '/' + std::string(db_name) +
5614 '/' + file_name : src_path;
5615
5616 size_t index = dest_path.find(".new");
5617 DBUG_ASSERT(index != std::string::npos);
5618 dest_path.replace(index, strlen(".ibd"), ".ibd");
5619 rename_force(src_path.c_str(),dest_path.c_str());
5620
5621 if (dest_dir) {
5622 /* remove delta and meta files to avoid delta applying for new file */
5623 index = src_path.find(".new");
5624 DBUG_ASSERT(index != std::string::npos);
5625 src_path.replace(index, std::string::npos, ".ibd.delta");
5626 if (access(src_path.c_str(), R_OK) == 0) {
5627 msg("Removing %s", src_path.c_str());
5628 if (my_delete(src_path.c_str(), MYF(MY_WME)))
5629 die("Can't remove %s, errno %d", src_path.c_str(), errno);
5630 }
5631 src_path.replace(index, std::string::npos, ".ibd.meta");
5632 if (access(src_path.c_str(), R_OK) == 0) {
5633 msg("Removing %s", src_path.c_str());
5634 if (my_delete(src_path.c_str(), MYF(MY_WME)))
5635 die("Can't remove %s, errno %d", src_path.c_str(), errno);
5636 }
5637
5638 /* add table name to the container to avoid it's deletion at the end of
5639 prepare */
5640 std::string table_name = std::string(db_name) + '/'
5641 + std::string(file_name, file_name + strlen(file_name) - strlen(".new"));
5642 xb_filter_entry_t *table = static_cast<xb_filter_entry_t *>
5643 (malloc(sizeof(xb_filter_entry_t) + table_name.size() + 1));
5644 table->name = ((char*)table) + sizeof(xb_filter_entry_t);
5645 strcpy(table->name, table_name.c_str());
5646 HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash,
5647 ut_fold_string(table->name), table);
5648 }
5649
5650 return TRUE;
5651 }
5652
5653 /************************************************************************
5654 Callback to handle datadir entry. Deletes entry if it has no matching
5655 fil_space in fil_system directory.
5656 @return FALSE if delete attempt was unsuccessful */
5657 static
5658 ibool
rm_if_not_found(const char * data_home_dir,const char * db_name,const char * file_name,void * arg)5659 rm_if_not_found(
5660 const char* data_home_dir, /*!<in: path to datadir */
5661 const char* db_name, /*!<in: database name */
5662 const char* file_name, /*!<in: file name with suffix */
5663 void* arg __attribute__((unused)))
5664 {
5665 char name[FN_REFLEN];
5666 xb_filter_entry_t* table;
5667
5668 snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name);
5669 /* Truncate ".ibd" */
5670 name[strlen(name) - 4] = '\0';
5671
5672 HASH_SEARCH(name_hash, inc_dir_tables_hash, ut_fold_string(name),
5673 xb_filter_entry_t*,
5674 table, (void) 0,
5675 !strcmp(table->name, name));
5676
5677 if (!table) {
5678 snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir,
5679 db_name, file_name);
5680 return os_file_delete(0, name);
5681 }
5682
5683 return(TRUE);
5684 }
5685
5686 /** Function enumerates files in datadir (provided by path) which are matched
5687 by provided suffix. For each entry callback is called.
5688
5689 @param[in] path datadir path
5690 @param[in] suffix suffix to match against
5691 @param[in] func callback
5692 @param[in] func_arg arguments for the above callback
5693
5694 @return FALSE if callback for some entry returned FALSE */
xb_process_datadir(const char * path,const char * suffix,handle_datadir_entry_func_t func,void * func_arg=NULL)5695 static ibool xb_process_datadir(const char *path, const char *suffix,
5696 handle_datadir_entry_func_t func,
5697 void *func_arg = NULL)
5698 {
5699 ulint ret;
5700 char dbpath[OS_FILE_MAX_PATH+2];
5701 os_file_dir_t dir;
5702 os_file_dir_t dbdir;
5703 os_file_stat_t dbinfo;
5704 os_file_stat_t fileinfo;
5705 ulint suffix_len;
5706 dberr_t err = DB_SUCCESS;
5707 static char current_dir[2];
5708
5709 current_dir[0] = FN_CURLIB;
5710 current_dir[1] = 0;
5711 srv_data_home = current_dir;
5712
5713 suffix_len = strlen(suffix);
5714
5715 /* datafile */
5716 dbdir = os_file_opendir(path);
5717 if (UNIV_UNLIKELY(dbdir != IF_WIN(INVALID_HANDLE_VALUE, nullptr))) {
5718 ret = fil_file_readdir_next_file(&err, path, dbdir, &fileinfo);
5719 while (ret == 0) {
5720 if (fileinfo.type == OS_FILE_TYPE_DIR) {
5721 goto next_file_item_1;
5722 }
5723
5724 if (strlen(fileinfo.name) > suffix_len
5725 && 0 == strcmp(fileinfo.name +
5726 strlen(fileinfo.name) - suffix_len,
5727 suffix)) {
5728 if (!func(
5729 path, NULL,
5730 fileinfo.name, func_arg))
5731 {
5732 os_file_closedir(dbdir);
5733 return(FALSE);
5734 }
5735 }
5736 next_file_item_1:
5737 ret = fil_file_readdir_next_file(&err,
5738 path, dbdir,
5739 &fileinfo);
5740 }
5741
5742 os_file_closedir(dbdir);
5743 } else {
5744 msg("Can't open dir %s", path);
5745 }
5746
5747 /* single table tablespaces */
5748 dir = os_file_opendir(path);
5749
5750 if (UNIV_UNLIKELY(dbdir == IF_WIN(INVALID_HANDLE_VALUE, nullptr))) {
5751 msg("Can't open dir %s", path);
5752 return TRUE;
5753 }
5754
5755 ret = fil_file_readdir_next_file(&err, path, dir, &dbinfo);
5756 while (ret == 0) {
5757 if (dbinfo.type == OS_FILE_TYPE_FILE
5758 || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
5759
5760 goto next_datadir_item;
5761 }
5762
5763 snprintf(dbpath, sizeof(dbpath), "%.*s/%.*s",
5764 OS_FILE_MAX_PATH/2-1,
5765 path,
5766 OS_FILE_MAX_PATH/2-1,
5767 dbinfo.name);
5768
5769 os_normalize_path(dbpath);
5770
5771 dbdir = os_file_opendir(dbpath);
5772
5773 if (dbdir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
5774 ret = fil_file_readdir_next_file(&err, dbpath, dbdir,
5775 &fileinfo);
5776 while (ret == 0) {
5777
5778 if (fileinfo.type == OS_FILE_TYPE_DIR) {
5779
5780 goto next_file_item_2;
5781 }
5782
5783 if (strlen(fileinfo.name) > suffix_len
5784 && 0 == strcmp(fileinfo.name +
5785 strlen(fileinfo.name) -
5786 suffix_len,
5787 suffix)) {
5788 /* The name ends in suffix; process
5789 the file */
5790 if (!func(
5791 path,
5792 dbinfo.name,
5793 fileinfo.name, func_arg))
5794 {
5795 os_file_closedir(dbdir);
5796 os_file_closedir(dir);
5797 return(FALSE);
5798 }
5799 }
5800 next_file_item_2:
5801 ret = fil_file_readdir_next_file(&err,
5802 dbpath, dbdir,
5803 &fileinfo);
5804 }
5805
5806 os_file_closedir(dbdir);
5807 }
5808 next_datadir_item:
5809 ret = fil_file_readdir_next_file(&err,
5810 path,
5811 dir, &dbinfo);
5812 }
5813
5814 os_file_closedir(dir);
5815
5816 return(TRUE);
5817 }
5818
5819 /************************************************************************
5820 Applies all .delta files from incremental_dir to the full backup.
5821 @return TRUE on success. */
5822 static
5823 ibool
xtrabackup_apply_deltas()5824 xtrabackup_apply_deltas()
5825 {
5826 return xb_process_datadir(xtrabackup_incremental_dir, ".delta",
5827 xtrabackup_apply_delta);
5828 }
5829
5830
5831 static
5832 void
innodb_free_param()5833 innodb_free_param()
5834 {
5835 srv_sys_space.shutdown();
5836 free_tmpdir(&mysql_tmpdir_list);
5837 }
5838
5839
5840 /** Check if file exists*/
file_exists(std::string name)5841 static bool file_exists(std::string name)
5842 {
5843 return access(name.c_str(), R_OK) == 0 ;
5844 }
5845
5846 /** Read file content into STL string */
read_file_as_string(const std::string file)5847 static std::string read_file_as_string(const std::string file) {
5848 char content[FN_REFLEN];
5849 FILE *f = fopen(file.c_str(), "r");
5850 if (!f) {
5851 msg("Can not open %s", file.c_str());
5852 }
5853 size_t len = fread(content, 1, FN_REFLEN, f);
5854 fclose(f);
5855 return std::string(content, len);
5856 }
5857
5858 /** Delete file- Provide verbose diagnostics and exit, if operation fails. */
delete_file(const std::string & file,bool if_exists=false)5859 static void delete_file(const std::string& file, bool if_exists = false) {
5860 if (if_exists && !file_exists(file))
5861 return;
5862 if (my_delete(file.c_str(), MYF(MY_WME))) {
5863 die("Can't remove %s, errno %d", file.c_str(), errno);
5864 }
5865 }
5866
5867 /**
5868 Rename tablespace during prepare.
5869 Backup in its end phase may generate some .ren files, recording
5870 tablespaces that should be renamed in --prepare.
5871 */
rename_table_in_prepare(const std::string & datadir,const std::string & from,const std::string & to,const char * extension=0)5872 static void rename_table_in_prepare(const std::string &datadir, const std::string& from , const std::string& to,
5873 const char *extension=0) {
5874 if (!extension) {
5875 static const char *extensions_nonincremental[] = { ".ibd", 0 };
5876 static const char *extensions_incremental[] = { ".ibd.delta", ".ibd.meta", 0 };
5877 const char **extensions = xtrabackup_incremental_dir ?
5878 extensions_incremental : extensions_nonincremental;
5879 for (size_t i = 0; extensions[i]; i++) {
5880 rename_table_in_prepare(datadir, from, to, extensions[i]);
5881 }
5882 return;
5883 }
5884 std::string src = std::string(datadir) + "/" + from + extension;
5885 std::string dest = std::string(datadir) + "/" + to + extension;
5886 std::string ren2, tmp;
5887 if (file_exists(dest)) {
5888 ren2= std::string(datadir) + "/" + to + ".ren";
5889 if (!file_exists(ren2)) {
5890 msg("ERROR : File %s was not found, but expected during rename processing\n", ren2.c_str());
5891 ut_a(0);
5892 }
5893 tmp = to + "#";
5894 rename_table_in_prepare(datadir, to, tmp);
5895 }
5896 rename_file(src, dest);
5897 if (ren2.size()) {
5898 // Make sure the temp. renamed file is processed.
5899 std::string to2 = read_file_as_string(ren2);
5900 rename_table_in_prepare(datadir, tmp, to2);
5901 delete_file(ren2);
5902 }
5903 }
5904
prepare_handle_ren_files(const char * datadir,const char * db,const char * filename,void *)5905 static ibool prepare_handle_ren_files(const char *datadir, const char *db, const char *filename, void *) {
5906
5907 std::string ren_file = std::string(datadir) + "/" + db + "/" + filename;
5908 if (!file_exists(ren_file))
5909 return TRUE;
5910
5911 std::string to = read_file_as_string(ren_file);
5912 std::string source_space_name = std::string(db) + "/" + filename;
5913 source_space_name.resize(source_space_name.size() - 4); // remove extension
5914
5915 rename_table_in_prepare(datadir, source_space_name.c_str(), to.c_str());
5916 delete_file(ren_file);
5917 return TRUE;
5918 }
5919
5920 /* Remove tablespaces during backup, based on */
prepare_handle_del_files(const char * datadir,const char * db,const char * filename,void *)5921 static ibool prepare_handle_del_files(const char *datadir, const char *db, const char *filename, void *) {
5922 std::string del_file = std::string(datadir) + "/" + db + "/" + filename;
5923 std::string path(del_file);
5924 path.resize(path.size() - 4); // remove extension;
5925 if (xtrabackup_incremental) {
5926 delete_file(path + ".ibd.delta", true);
5927 delete_file(path + ".ibd.meta", true);
5928 }
5929 else {
5930 delete_file(path + ".ibd", true);
5931 }
5932 delete_file(del_file);
5933 return TRUE;
5934 }
5935
5936 /** Implement --prepare
5937 @return whether the operation succeeded */
xtrabackup_prepare_func(char ** argv)5938 static bool xtrabackup_prepare_func(char** argv)
5939 {
5940 CorruptedPages corrupted_pages;
5941 char metadata_path[FN_REFLEN];
5942
5943 /* cd to target-dir */
5944
5945 if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME)))
5946 {
5947 msg("can't my_setwd %s", xtrabackup_real_target_dir);
5948 return(false);
5949 }
5950 msg("cd to %s", xtrabackup_real_target_dir);
5951
5952 fil_path_to_mysql_datadir = ".";
5953
5954 ut_ad(xtrabackup_incremental == xtrabackup_incremental_dir);
5955 if (xtrabackup_incremental) {
5956 inc_dir_tables_hash = hash_create(1000);
5957 ut_ad(inc_dir_tables_hash);
5958 }
5959
5960 msg("open files limit requested %u, set to %u",
5961 (uint) xb_open_files_limit,
5962 xb_set_max_open_files(xb_open_files_limit));
5963
5964 /* Fix DDL for prepare. Process .del,.ren, and .new files.
5965 The order in which files are processed, is important
5966 (see MDEV-18185, MDEV-18201)
5967 */
5968 xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".",
5969 ".del", prepare_handle_del_files);
5970 xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".",
5971 ".ren", prepare_handle_ren_files);
5972 if (xtrabackup_incremental_dir) {
5973 xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files);
5974 xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files);
5975 xb_process_datadir(xtrabackup_incremental_dir, ".new",
5976 prepare_handle_new_files, (void *)".");
5977 }
5978 else {
5979 xb_process_datadir(".", ".new", prepare_handle_new_files);
5980 }
5981
5982 int argc; for (argc = 0; argv[argc]; argc++) {}
5983 encryption_plugin_prepare_init(argc, argv);
5984
5985 xtrabackup_target_dir= mysql_data_home_buff;
5986 xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here
5987 xtrabackup_target_dir[1]=0;
5988 const lsn_t target_lsn = xtrabackup_incremental
5989 ? incremental_to_lsn : metadata_to_lsn;
5990
5991 /*
5992 read metadata of target
5993 */
5994 sprintf(metadata_path, "%s/%s", xtrabackup_target_dir,
5995 XTRABACKUP_METADATA_FILENAME);
5996
5997 if (!xtrabackup_read_metadata(metadata_path)) {
5998 msg("Error: failed to read metadata from '%s'\n",
5999 metadata_path);
6000 return(false);
6001 }
6002
6003 if (!strcmp(metadata_type, "full-backuped")) {
6004 if (xtrabackup_incremental) {
6005 msg("error: applying incremental backup "
6006 "needs a prepared target.");
6007 return(false);
6008 }
6009 msg("This target seems to be not prepared yet.");
6010 } else if (!strcmp(metadata_type, "log-applied")) {
6011 msg("This target seems to be already prepared.");
6012 } else {
6013 msg("This target does not have correct metadata.");
6014 return(false);
6015 }
6016
6017 bool ok = !xtrabackup_incremental
6018 || metadata_to_lsn == incremental_lsn;
6019 if (!ok) {
6020 msg("error: This incremental backup seems "
6021 "not to be proper for the target. Check 'to_lsn' of the target and "
6022 "'from_lsn' of the incremental.");
6023 return(false);
6024 }
6025
6026 srv_max_n_threads = 1000;
6027 srv_undo_logs = 1;
6028 srv_n_purge_threads = 1;
6029
6030 xb_filters_init();
6031
6032 srv_log_group_home_dir = NULL;
6033 srv_thread_concurrency = 1;
6034
6035 if (xtrabackup_incremental) {
6036 srv_operation = SRV_OPERATION_RESTORE_DELTA;
6037
6038 if (innodb_init_param()) {
6039 goto error_cleanup;
6040 }
6041
6042 sync_check_init();
6043 ut_d(sync_check_enable());
6044 ut_crc32_init();
6045 recv_sys.create();
6046 log_sys.create();
6047 recv_recovery_on = true;
6048
6049 #ifdef WITH_INNODB_DISALLOW_WRITES
6050 srv_allow_writes_event = os_event_create(0);
6051 os_event_set(srv_allow_writes_event);
6052 #endif
6053 dberr_t err = xb_data_files_init();
6054 if (err != DB_SUCCESS) {
6055 msg("mariabackup: error: xb_data_files_init() failed "
6056 "with error %s\n", ut_strerr(err));
6057 goto error_cleanup;
6058 }
6059
6060 ok = xtrabackup_apply_deltas();
6061
6062 xb_data_files_close();
6063
6064 if (ok) {
6065 /* Cleanup datadir from tablespaces deleted
6066 between full and incremental backups */
6067
6068 xb_process_datadir("./", ".ibd", rm_if_not_found);
6069 }
6070
6071 xb_filter_hash_free(inc_dir_tables_hash);
6072
6073 fil_system.close();
6074 #ifdef WITH_INNODB_DISALLOW_WRITES
6075 os_event_destroy(srv_allow_writes_event);
6076 #endif
6077 innodb_free_param();
6078 log_sys.close();
6079 sync_check_close();
6080 if (!ok) goto error_cleanup;
6081 }
6082
6083 srv_operation=
6084 xtrabackup_export
6085 ? SRV_OPERATION_RESTORE_EXPORT
6086 : (xtrabackup_rollback_xa ? SRV_OPERATION_RESTORE_ROLLBACK_XA
6087 : SRV_OPERATION_RESTORE);
6088
6089 if (innodb_init_param()) {
6090 goto error_cleanup;
6091 }
6092
6093 /* increase IO threads */
6094 if (srv_n_file_io_threads < 10) {
6095 srv_n_read_io_threads = 4;
6096 srv_n_write_io_threads = 4;
6097 }
6098
6099 msg("Starting InnoDB instance for recovery.");
6100
6101 msg("mariabackup: Using %lld bytes for buffer pool "
6102 "(set by --use-memory parameter)", xtrabackup_use_memory);
6103
6104 srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct;
6105
6106 if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) {
6107 srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct;
6108 }
6109
6110 if (xtrabackup_rollback_xa)
6111 srv_fast_shutdown= 0;
6112
6113 if (innodb_init()) {
6114 goto error_cleanup;
6115 }
6116
6117 corrupted_pages.read_from_file(MB_CORRUPTED_PAGES_FILE);
6118 if (xtrabackup_incremental)
6119 {
6120 char inc_filename[FN_REFLEN];
6121 sprintf(inc_filename, "%s/%s", xtrabackup_incremental_dir,
6122 MB_CORRUPTED_PAGES_FILE);
6123 corrupted_pages.read_from_file(inc_filename);
6124 }
6125 if (!corrupted_pages.empty())
6126 corrupted_pages.zero_out_free_pages();
6127 if (corrupted_pages.empty())
6128 {
6129 if (!xtrabackup_incremental && unlink(MB_CORRUPTED_PAGES_FILE) &&
6130 errno != ENOENT)
6131 {
6132 char errbuf[MYSYS_STRERROR_SIZE];
6133 my_strerror(errbuf, sizeof(errbuf), errno);
6134 die("Error: unlink %s failed: %s", MB_CORRUPTED_PAGES_FILE,
6135 errbuf);
6136 }
6137 }
6138 else
6139 corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE);
6140
6141 if (xtrabackup_rollback_xa)
6142 {
6143 /* Please do not merge MDEV-21168 fix in 10.5+ */
6144 compile_time_assert(MYSQL_VERSION_ID < 10 * 10000 + 5 * 100);
6145 XID *xid_list=
6146 (XID *) my_malloc(MAX_XID_LIST_SIZE * sizeof(XID), MYF(0));
6147 if (!xid_list)
6148 {
6149 msg("Can't allocate %i bytes for XID's list", MAX_XID_LIST_SIZE);
6150 ok= false;
6151 goto error_cleanup;
6152 }
6153 int got;
6154 ut_ad(recv_no_log_write);
6155 ut_d(recv_no_log_write= false);
6156 while ((got= trx_recover_for_mysql(xid_list, MAX_XID_LIST_SIZE)) > 0)
6157 {
6158 for (int i= 0; i < got; i++)
6159 {
6160 #ifndef DBUG_OFF
6161 int rc=
6162 #endif // !DBUG_OFF
6163 innobase_rollback_by_xid(NULL, xid_list + i);
6164 #ifndef DBUG_OFF
6165 if (rc == 0)
6166 {
6167 char buf[XIDDATASIZE * 4 + 6]; // see xid_to_str
6168 DBUG_PRINT("info",
6169 ("rollback xid %s", xid_to_str(buf, xid_list[i])));
6170 }
6171 #endif // !DBUG_OFF
6172 }
6173 }
6174 ut_d(recv_no_log_write= true);
6175 my_free(xid_list);
6176 }
6177
6178 if (ok) {
6179 msg("Last binlog file %s, position %lld",
6180 trx_sys.recovered_binlog_filename,
6181 longlong(trx_sys.recovered_binlog_offset));
6182 }
6183
6184 /* Check whether the log is applied enough or not. */
6185 if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID))
6186 && srv_start_lsn < target_lsn) {
6187 msg("mariabackup: error: "
6188 "The log was only applied up to LSN " LSN_PF
6189 ", instead of " LSN_PF,
6190 srv_start_lsn, target_lsn);
6191 ok = false;
6192 }
6193 #ifdef WITH_WSREP
6194 else if (ok) xb_write_galera_info(xtrabackup_incremental);
6195 #endif
6196
6197 if (xtrabackup_rollback_xa)
6198 {
6199 // See innobase_end() and thd_destructor_proxy()
6200 while (srv_fast_shutdown == 0 &&
6201 (trx_sys.any_active_transactions() ||
6202 THD_count::value() > srv_n_purge_threads + 1))
6203 os_thread_sleep(1000);
6204
6205 srv_shutdown_bg_undo_sources();
6206 srv_purge_shutdown();
6207 buf_flush_sync_all_buf_pools();
6208 }
6209
6210 innodb_shutdown();
6211 innodb_free_param();
6212
6213 /* output to metadata file */
6214 if (ok) {
6215 char filename[FN_REFLEN];
6216
6217 strcpy(metadata_type, "log-applied");
6218
6219 if(xtrabackup_incremental
6220 && metadata_to_lsn < incremental_to_lsn)
6221 {
6222 metadata_to_lsn = incremental_to_lsn;
6223 metadata_last_lsn = incremental_last_lsn;
6224 }
6225
6226 sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
6227 if (!xtrabackup_write_metadata(filename)) {
6228
6229 msg("mariabackup: Error: failed to write metadata "
6230 "to '%s'", filename);
6231 ok = false;
6232 } else if (xtrabackup_extra_lsndir) {
6233 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
6234 if (!xtrabackup_write_metadata(filename)) {
6235 msg("mariabackup: Error: failed to write "
6236 "metadata to '%s'", filename);
6237 ok = false;
6238 }
6239 }
6240 }
6241
6242 if (ok) ok = apply_log_finish();
6243
6244 if (ok && xtrabackup_export)
6245 ok= (prepare_export() == 0);
6246
6247 error_cleanup:
6248 xb_filters_free();
6249 return ok && !ib::error::was_logged() && corrupted_pages.empty();
6250 }
6251
6252 /**************************************************************************
6253 Append group name to xb_load_default_groups list. */
6254 static
6255 void
append_defaults_group(const char * group,const char * default_groups[],size_t default_groups_size)6256 append_defaults_group(const char *group, const char *default_groups[],
6257 size_t default_groups_size)
6258 {
6259 uint i;
6260 bool appended = false;
6261 for (i = 0; i < default_groups_size - 1; i++) {
6262 if (default_groups[i] == NULL) {
6263 default_groups[i] = group;
6264 appended = true;
6265 break;
6266 }
6267 }
6268 ut_a(appended);
6269 }
6270
6271 static const char*
normalize_privilege_target_name(const char * name)6272 normalize_privilege_target_name(const char* name)
6273 {
6274 if (strcmp(name, "*") == 0) {
6275 return "\\*";
6276 }
6277 else {
6278 /* should have no regex special characters. */
6279 ut_ad(strpbrk(name, ".()[]*+?") == 0);
6280 }
6281 return name;
6282 }
6283
6284 /******************************************************************//**
6285 Check if specific privilege is granted.
6286 Uses regexp magic to check if requested privilege is granted for given
6287 database.table or database.* or *.*
6288 or if user has 'ALL PRIVILEGES' granted.
6289 @return true if requested privilege is granted, false otherwise. */
6290 static bool
has_privilege(const std::list<std::string> & granted,const char * required,const char * db_name,const char * table_name)6291 has_privilege(const std::list<std::string> &granted,
6292 const char* required,
6293 const char* db_name,
6294 const char* table_name)
6295 {
6296 char buffer[1000];
6297 regex_t priv_re;
6298 regmatch_t tables_regmatch[1];
6299 bool result = false;
6300
6301 db_name = normalize_privilege_target_name(db_name);
6302 table_name = normalize_privilege_target_name(table_name);
6303
6304 int written = snprintf(buffer, sizeof(buffer),
6305 "GRANT .*(%s)|(ALL PRIVILEGES).* ON (\\*|`%s`)\\.(\\*|`%s`)",
6306 required, db_name, table_name);
6307 if (written < 0 || written == sizeof(buffer)
6308 || regcomp(&priv_re, buffer, REG_EXTENDED)) {
6309 die("regcomp() failed for '%s'", buffer);
6310 }
6311
6312 typedef std::list<std::string>::const_iterator string_iter;
6313 for (string_iter i = granted.begin(), e = granted.end(); i != e; ++i) {
6314 int res = regexec(&priv_re, i->c_str(),
6315 1, tables_regmatch, 0);
6316
6317 if (res != REG_NOMATCH) {
6318 result = true;
6319 break;
6320 }
6321 }
6322
6323 xb_regfree(&priv_re);
6324 return result;
6325 }
6326
6327 enum {
6328 PRIVILEGE_OK = 0,
6329 PRIVILEGE_WARNING = 1,
6330 PRIVILEGE_ERROR = 2,
6331 };
6332
6333 /******************************************************************//**
6334 Check if specific privilege is granted.
6335 Prints error message if required privilege is missing.
6336 @return PRIVILEGE_OK if requested privilege is granted, error otherwise. */
6337 static
check_privilege(const std::list<std::string> & granted_priv,const char * required,const char * target_database,const char * target_table,int error=PRIVILEGE_ERROR)6338 int check_privilege(
6339 const std::list<std::string> &granted_priv, /* in: list of
6340 granted privileges*/
6341 const char* required, /* in: required privilege name */
6342 const char* target_database, /* in: required privilege target
6343 database name */
6344 const char* target_table, /* in: required privilege target
6345 table name */
6346 int error = PRIVILEGE_ERROR) /* in: return value if privilege
6347 is not granted */
6348 {
6349 if (!has_privilege(granted_priv,
6350 required, target_database, target_table)) {
6351 msg("%s: missing required privilege %s on %s.%s",
6352 (error == PRIVILEGE_ERROR ? "Error" : "Warning"),
6353 required, target_database, target_table);
6354 return error;
6355 }
6356 return PRIVILEGE_OK;
6357 }
6358
6359
6360 /**
6361 Check DB user privileges according to the intended actions.
6362
6363 Fetches DB user privileges, determines intended actions based on
6364 command-line arguments and prints missing privileges.
6365 @return whether all the necessary privileges are granted */
check_all_privileges()6366 static bool check_all_privileges()
6367 {
6368 if (!mysql_connection) {
6369 /* Not connected, no queries is going to be executed. */
6370 return true;
6371 }
6372
6373 /* Fetch effective privileges. */
6374 std::list<std::string> granted_privileges;
6375 MYSQL_RES* result = xb_mysql_query(mysql_connection, "SHOW GRANTS",
6376 true);
6377 while (MYSQL_ROW row = mysql_fetch_row(result)) {
6378 granted_privileges.push_back(*row);
6379 }
6380 mysql_free_result(result);
6381
6382 int check_result = PRIVILEGE_OK;
6383
6384 /* FLUSH TABLES WITH READ LOCK */
6385 if (!opt_no_lock)
6386 {
6387 check_result |= check_privilege(
6388 granted_privileges,
6389 "RELOAD", "*", "*");
6390 check_result |= check_privilege(
6391 granted_privileges,
6392 "PROCESS", "*", "*");
6393 }
6394
6395 /* KILL ... */
6396 if ((!opt_no_lock && (opt_kill_long_queries_timeout || opt_lock_ddl_per_table))
6397 /* START SLAVE SQL_THREAD */
6398 /* STOP SLAVE SQL_THREAD */
6399 || opt_safe_slave_backup) {
6400 check_result |= check_privilege(
6401 granted_privileges,
6402 "SUPER", "*", "*",
6403 PRIVILEGE_WARNING);
6404 }
6405
6406 /* SHOW MASTER STATUS */
6407 /* SHOW SLAVE STATUS */
6408 if (opt_galera_info || opt_slave_info
6409 || (opt_no_lock && opt_safe_slave_backup)) {
6410 check_result |= check_privilege(granted_privileges,
6411 "REPLICATION CLIENT", "*", "*",
6412 PRIVILEGE_WARNING);
6413 }
6414
6415 if (check_result & PRIVILEGE_ERROR) {
6416 msg("Current privileges, as reported by 'SHOW GRANTS': ");
6417 int n=1;
6418 for (std::list<std::string>::const_iterator it = granted_privileges.begin();
6419 it != granted_privileges.end();
6420 it++,n++) {
6421 msg(" %d.%s", n, it->c_str());
6422 }
6423 return false;
6424 }
6425
6426 return true;
6427 }
6428
6429 bool
xb_init()6430 xb_init()
6431 {
6432 const char *mixed_options[4] = {NULL, NULL, NULL, NULL};
6433 int n_mixed_options;
6434
6435 /* sanity checks */
6436
6437 if (opt_slave_info
6438 && opt_no_lock
6439 && !opt_safe_slave_backup) {
6440 msg("Error: --slave-info is used with --no-lock but "
6441 "without --safe-slave-backup. The binlog position "
6442 "cannot be consistent with the backup data.");
6443 return(false);
6444 }
6445
6446 if (xtrabackup_backup && opt_rsync)
6447 {
6448 if (xtrabackup_stream_fmt)
6449 {
6450 msg("Error: --rsync doesn't work with --stream\n");
6451 return(false);
6452 }
6453 bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0));
6454 if (!have_rsync)
6455 {
6456 msg("Error: rsync executable not found, cannot run backup with --rsync\n");
6457 return false;
6458 }
6459 }
6460
6461 n_mixed_options = 0;
6462
6463 if (opt_decompress) {
6464 mixed_options[n_mixed_options++] = "--decompress";
6465 }
6466
6467 if (xtrabackup_copy_back) {
6468 mixed_options[n_mixed_options++] = "--copy-back";
6469 }
6470
6471 if (xtrabackup_move_back) {
6472 mixed_options[n_mixed_options++] = "--move-back";
6473 }
6474
6475 if (xtrabackup_prepare) {
6476 mixed_options[n_mixed_options++] = "--apply-log";
6477 }
6478
6479 if (n_mixed_options > 1) {
6480 msg("Error: %s and %s are mutually exclusive\n",
6481 mixed_options[0], mixed_options[1]);
6482 return(false);
6483 }
6484
6485 if (xtrabackup_backup) {
6486 if ((mysql_connection = xb_mysql_connect()) == NULL) {
6487 return(false);
6488 }
6489
6490 if (!get_mysql_vars(mysql_connection)) {
6491 return(false);
6492 }
6493 if (opt_check_privileges && !check_all_privileges()) {
6494 return(false);
6495 }
6496 history_start_time = time(NULL);
6497
6498 }
6499
6500 return(true);
6501 }
6502
6503
6504 extern void init_signals(void);
6505
6506 #include <sql_locale.h>
6507
6508
setup_error_messages()6509 void setup_error_messages()
6510 {
6511 my_default_lc_messages = &my_locale_en_US;
6512 if (init_errmessage())
6513 die("could not initialize error messages");
6514 }
6515
6516 /** Handle mariabackup options. The options are handled with the following
6517 order:
6518
6519 1) Load server groups and process server options, ignore unknown options
6520 2) Load client groups and process client options, ignore unknown options
6521 3) Load backup groups and process client-server options, exit on unknown option
6522 4) Process --mysqld-args options, ignore unknown options
6523
6524 @param[in] argc arguments count
6525 @param[in] argv arguments array
6526 @param[out] argv_server server options including loaded from server groups
6527 @param[out] argv_client client options including loaded from client groups
6528 @param[out] argv_backup backup options including loaded from backup groups */
handle_options(int argc,char ** argv,char *** argv_server,char *** argv_client,char *** argv_backup)6529 void handle_options(int argc, char **argv, char ***argv_server,
6530 char ***argv_client, char ***argv_backup)
6531 {
6532 /* Setup some variables for Innodb.*/
6533 srv_operation = SRV_OPERATION_RESTORE;
6534
6535 files_charset_info = &my_charset_utf8_general_ci;
6536
6537
6538 setup_error_messages();
6539 sys_var_init();
6540 plugin_mutex_init();
6541 mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
6542 opt_stack_trace = 1;
6543 test_flags |= TEST_SIGINT;
6544 init_signals();
6545 #ifndef _WIN32
6546 /* Exit process on SIGINT. */
6547 my_sigset(SIGINT, SIG_DFL);
6548 #endif
6549
6550 sf_leaking_memory = 1; /* don't report memory leaks on early exist */
6551
6552 int i;
6553 int ho_error;
6554
6555 char* target_dir = NULL;
6556 bool prepare = false;
6557
6558 char conf_file[FN_REFLEN];
6559
6560 // array_elements() will not work for load_defaults, as it is defined
6561 // as external symbol, so let's use dynamic array to have ability to
6562 // add new server default groups
6563 std::vector<const char *> server_default_groups;
6564
6565 for (const char **default_group= load_default_groups; *default_group;
6566 ++default_group)
6567 server_default_groups.push_back(*default_group);
6568
6569 std::vector<char *> mysqld_args;
6570 std::vector<char *> mariabackup_args;
6571 mysqld_args.push_back(argv[0]);
6572 mariabackup_args.push_back(argv[0]);
6573
6574 /* scan options for group and config file to load defaults from */
6575 for (i= 1; i < argc; i++)
6576 {
6577 char *optend= strcend(argv[i], '=');
6578 if (mysqld_args.size() > 1 ||
6579 strncmp(argv[i], "--mysqld-args", optend - argv[i]) == 0)
6580 {
6581 mysqld_args.push_back(argv[i]);
6582 continue;
6583 }
6584 else
6585 mariabackup_args.push_back(argv[i]);
6586
6587 if (strncmp(argv[i], "--defaults-group", optend - argv[i]) == 0)
6588 {
6589 defaults_group= optend + 1;
6590 server_default_groups.push_back(defaults_group);
6591 }
6592 else if (strncmp(argv[i], "--login-path", optend - argv[i]) == 0)
6593 {
6594 append_defaults_group(optend + 1, xb_client_default_groups,
6595 array_elements(xb_client_default_groups));
6596 }
6597 else if (!strncmp(argv[i], "--prepare", optend - argv[i]))
6598 {
6599 prepare= true;
6600 }
6601 else if (!strncmp(argv[i], "--apply-log", optend - argv[i]))
6602 {
6603 prepare= true;
6604 }
6605 else if (!strncmp(argv[i], "--target-dir", optend - argv[i]) &&
6606 *optend)
6607 {
6608 target_dir= optend + 1;
6609 }
6610 else if (!*optend && argv[i][0] != '-')
6611 {
6612 target_dir= argv[i];
6613 }
6614 }
6615
6616 server_default_groups.push_back(NULL);
6617 snprintf(conf_file, sizeof(conf_file), "my");
6618
6619 if (prepare && target_dir) {
6620 snprintf(conf_file, sizeof(conf_file),
6621 "%s/backup-my.cnf", target_dir);
6622 if (!strncmp(argv[1], "--defaults-file=", 16)) {
6623 /* Remove defaults-file*/
6624 for (int i = 2; ; i++) {
6625 if ((argv[i-1]= argv[i]) == 0)
6626 break;
6627 }
6628 argc--;
6629 }
6630 }
6631
6632 mariabackup_args.push_back(nullptr);
6633 *argv_client= *argv_server= *argv_backup= &mariabackup_args[0];
6634 int argc_backup= static_cast<int>(mariabackup_args.size() - 1);
6635 int argc_client= argc_backup;
6636 int argc_server= argc_backup;
6637
6638 /* 1) Load server groups and process server options, ignore unknown
6639 options */
6640
6641 load_defaults_or_exit(conf_file, &server_default_groups[0],
6642 &argc_server, argv_server);
6643
6644 int n;
6645 for (n = 0; (*argv_server)[n]; n++) {};
6646 argc_server = n;
6647
6648 print_param_str <<
6649 "# This MySQL options file was generated by XtraBackup.\n"
6650 "[" << defaults_group << "]\n";
6651
6652 /* We want xtrabackup to ignore unknown options, because it only
6653 recognizes a small subset of server variables */
6654 my_getopt_skip_unknown = TRUE;
6655
6656 /* Reset u_max_value for all options, as we don't want the
6657 --maximum-... modifier to set the actual option values */
6658 for (my_option *optp= xb_server_options; optp->name; optp++) {
6659 optp->u_max_value = (G_PTR *) &global_max_value;
6660 }
6661
6662 /* Throw a descriptive error if --defaults-file or --defaults-extra-file
6663 is not the first command line argument */
6664 for (int i = 2 ; i < argc ; i++) {
6665 char *optend = strcend((argv)[i], '=');
6666
6667 if (optend - argv[i] == 15 &&
6668 !strncmp(argv[i], "--defaults-file", optend - argv[i])) {
6669 die("--defaults-file must be specified first on the command line");
6670 }
6671 if (optend - argv[i] == 21 &&
6672 !strncmp(argv[i], "--defaults-extra-file",
6673 optend - argv[i])) {
6674 die("--defaults-extra-file must be specified first on the command line");
6675 }
6676 }
6677
6678 if (argc_server > 0
6679 && (ho_error=handle_options(&argc_server, argv_server,
6680 xb_server_options, xb_get_one_option)))
6681 exit(ho_error);
6682
6683 /* 2) Load client groups and process client options, ignore unknown
6684 options */
6685
6686 load_defaults_or_exit(conf_file, xb_client_default_groups,
6687 &argc_client, argv_client);
6688
6689 for (n = 0; (*argv_client)[n]; n++) {};
6690 argc_client = n;
6691
6692 if (innobackupex_mode && argc_client > 0) {
6693 if (!ibx_handle_options(&argc_client, argv_client)) {
6694 exit(EXIT_FAILURE);
6695 }
6696 }
6697
6698 if (argc_client > 0
6699 && (ho_error=handle_options(&argc_client, argv_client,
6700 xb_client_options, xb_get_one_option)))
6701 exit(ho_error);
6702
6703 /* 3) Load backup groups and process client-server options, exit on
6704 unknown option */
6705
6706 load_defaults_or_exit(conf_file, backup_default_groups, &argc_backup,
6707 argv_backup);
6708 for (n= 0; (*argv_backup)[n]; n++)
6709 {
6710 };
6711 argc_backup= n;
6712
6713 my_handle_options_init_variables = FALSE;
6714
6715 if (argc_backup > 0 &&
6716 (ho_error= handle_options(&argc_backup, argv_backup,
6717 xb_server_options, xb_get_one_option)))
6718 exit(ho_error);
6719
6720 /* Add back the program name handle_options removes */
6721 ++argc_backup;
6722 --(*argv_backup);
6723
6724 if (innobackupex_mode && argc_backup > 0 &&
6725 !ibx_handle_options(&argc_backup, argv_backup))
6726 exit(EXIT_FAILURE);
6727
6728 my_getopt_skip_unknown = FALSE;
6729
6730 if (argc_backup > 0 &&
6731 (ho_error= handle_options(&argc_backup, argv_backup,
6732 xb_client_options, xb_get_one_option)))
6733 exit(ho_error);
6734
6735 if (opt_password)
6736 {
6737 char *argument= opt_password;
6738 char *start= argument;
6739 opt_password= my_strdup(opt_password, MYF(MY_FAE));
6740 while (*argument)
6741 *argument++= 'x'; // Destroy argument
6742 if (*start)
6743 start[1]= 0;
6744 }
6745
6746 /* 4) Process --mysqld-args options, ignore unknown options */
6747
6748 my_getopt_skip_unknown = TRUE;
6749
6750 int argc_mysqld = static_cast<int>(mysqld_args.size());
6751 if (argc_mysqld > 1)
6752 {
6753 char **argv_mysqld= &mysqld_args[0];
6754 if ((ho_error= handle_options(&argc_mysqld, &argv_mysqld,
6755 xb_server_options, xb_get_one_option)))
6756 exit(ho_error);
6757 }
6758
6759 my_handle_options_init_variables = TRUE;
6760
6761 /* Reject command line arguments that don't look like options, i.e. are
6762 not of the form '-X' (single-character options) or '--option' (long
6763 options) */
6764 for (int i = 0 ; i < argc_backup ; i++) {
6765 const char * const opt = (*argv_backup)[i];
6766
6767 if (strncmp(opt, "--", 2) &&
6768 !(strlen(opt) == 2 && opt[0] == '-')) {
6769 bool server_option = true;
6770
6771 for (int j = 0; j < argc_backup; j++) {
6772 if (opt == (*argv_backup)[j]) {
6773 server_option = false;
6774 break;
6775 }
6776 }
6777
6778 if (!server_option) {
6779 msg("mariabackup: Error:"
6780 " unknown argument: '%s'", opt);
6781 exit(EXIT_FAILURE);
6782 }
6783 }
6784 }
6785 }
6786
6787 static int main_low(char** argv);
6788 static int get_exepath(char *buf, size_t size, const char *argv0);
6789
6790 /* ================= main =================== */
main(int argc,char ** argv)6791 int main(int argc, char **argv)
6792 {
6793 char **server_defaults;
6794 char **client_defaults;
6795 char **backup_defaults;
6796
6797 my_getopt_prefix_matching= 0;
6798
6799 if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0]))
6800 strncpy(mariabackup_exe,argv[0], FN_REFLEN-1);
6801
6802
6803 if (argc > 1 )
6804 {
6805 /* In "prepare export", we need to start mysqld
6806 Since it is not always be installed on the machine,
6807 we start "mariabackup --mysqld", which acts as mysqld
6808 */
6809 if (strcmp(argv[1], "--mysqld") == 0)
6810 {
6811 extern int mysqld_main(int argc, char **argv);
6812 argc--;
6813 argv++;
6814 argv[0]+=2;
6815 return mysqld_main(argc, argv);
6816 }
6817 if(strcmp(argv[1], "--innobackupex") == 0)
6818 {
6819 argv++;
6820 argc--;
6821 innobackupex_mode = true;
6822 }
6823 }
6824
6825 if (argc > 1)
6826 strncpy(orig_argv1,argv[1],sizeof(orig_argv1) -1);
6827
6828 init_signals();
6829 MY_INIT(argv[0]);
6830
6831 pthread_key_create(&THR_THD, NULL);
6832 my_pthread_setspecific_ptr(THR_THD, NULL);
6833
6834 xb_regex_init();
6835
6836 capture_tool_command(argc, argv);
6837
6838 if (mysql_server_init(-1, NULL, NULL))
6839 {
6840 die("mysql_server_init() failed");
6841 }
6842
6843 system_charset_info = &my_charset_utf8_general_ci;
6844 key_map_full.set_all();
6845
6846 logger.init_base();
6847 logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE);
6848 mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log,
6849 MY_MUTEX_INIT_FAST);
6850
6851 handle_options(argc, argv, &server_defaults, &client_defaults,
6852 &backup_defaults);
6853
6854 #ifndef DBUG_OFF
6855 if (dbug_option) {
6856 DBUG_SET_INITIAL(dbug_option);
6857 DBUG_SET(dbug_option);
6858 }
6859 #endif
6860
6861 int status = main_low(server_defaults);
6862
6863 backup_cleanup();
6864
6865 if (innobackupex_mode) {
6866 ibx_cleanup();
6867 }
6868
6869 free_defaults(server_defaults);
6870 free_defaults(client_defaults);
6871 free_defaults(backup_defaults);
6872
6873 #ifndef DBUG_OFF
6874 if (dbug_option) {
6875 DBUG_END();
6876 }
6877 #endif
6878
6879 if (THR_THD)
6880 (void) pthread_key_delete(THR_THD);
6881
6882 logger.cleanup_base();
6883 cleanup_errmsgs();
6884 free_error_messages();
6885 mysql_mutex_destroy(&LOCK_error_log);
6886
6887 if (status == EXIT_SUCCESS) {
6888 msg("completed OK!");
6889 }
6890
6891 return status;
6892 }
6893
main_low(char ** argv)6894 static int main_low(char** argv)
6895 {
6896 if (innobackupex_mode) {
6897 if (!ibx_init()) {
6898 return(EXIT_FAILURE);
6899 }
6900 }
6901
6902 if (!xtrabackup_print_param && !xtrabackup_prepare
6903 && !strcmp(mysql_data_home, "./")) {
6904 if (!xtrabackup_print_param)
6905 usage();
6906 msg("mariabackup: Error: Please set parameter 'datadir'");
6907 return(EXIT_FAILURE);
6908 }
6909
6910 /* Expand target-dir, incremental-basedir, etc. */
6911
6912 char cwd[FN_REFLEN];
6913 my_getwd(cwd, sizeof(cwd), MYF(0));
6914
6915 my_load_path(xtrabackup_real_target_dir,
6916 xtrabackup_target_dir, cwd);
6917 unpack_dirname(xtrabackup_real_target_dir,
6918 xtrabackup_real_target_dir);
6919 xtrabackup_target_dir= xtrabackup_real_target_dir;
6920
6921 if (xtrabackup_incremental_basedir) {
6922 my_load_path(xtrabackup_real_incremental_basedir,
6923 xtrabackup_incremental_basedir, cwd);
6924 unpack_dirname(xtrabackup_real_incremental_basedir,
6925 xtrabackup_real_incremental_basedir);
6926 xtrabackup_incremental_basedir =
6927 xtrabackup_real_incremental_basedir;
6928 }
6929
6930 if (xtrabackup_incremental_dir) {
6931 my_load_path(xtrabackup_real_incremental_dir,
6932 xtrabackup_incremental_dir, cwd);
6933 unpack_dirname(xtrabackup_real_incremental_dir,
6934 xtrabackup_real_incremental_dir);
6935 xtrabackup_incremental_dir = xtrabackup_real_incremental_dir;
6936 }
6937
6938 if (xtrabackup_extra_lsndir) {
6939 my_load_path(xtrabackup_real_extra_lsndir,
6940 xtrabackup_extra_lsndir, cwd);
6941 unpack_dirname(xtrabackup_real_extra_lsndir,
6942 xtrabackup_real_extra_lsndir);
6943 xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir;
6944 }
6945
6946 /* get default temporary directory */
6947 if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
6948 opt_mysql_tmpdir = getenv("TMPDIR");
6949 #if defined(__WIN__)
6950 if (!opt_mysql_tmpdir) {
6951 opt_mysql_tmpdir = getenv("TEMP");
6952 }
6953 if (!opt_mysql_tmpdir) {
6954 opt_mysql_tmpdir = getenv("TMP");
6955 }
6956 #endif
6957 if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
6958 opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR);
6959 }
6960 }
6961
6962 /* temporary setting of enough size */
6963 srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX;
6964 srv_page_size = UNIV_PAGE_SIZE_MAX;
6965 if (xtrabackup_backup && xtrabackup_incremental) {
6966 /* direct specification is only for --backup */
6967 /* and the lsn is prior to the other option */
6968
6969 char* endchar;
6970 int error = 0;
6971 incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10);
6972 if (*endchar != '\0')
6973 error = 1;
6974
6975 if (error) {
6976 msg("mariabackup: value '%s' may be wrong format for "
6977 "incremental option.", xtrabackup_incremental);
6978 return(EXIT_FAILURE);
6979 }
6980 } else if (xtrabackup_backup && xtrabackup_incremental_basedir) {
6981 char filename[FN_REFLEN];
6982
6983 sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME);
6984
6985 if (!xtrabackup_read_metadata(filename)) {
6986 msg("mariabackup: error: failed to read metadata from "
6987 "%s", filename);
6988 return(EXIT_FAILURE);
6989 }
6990
6991 incremental_lsn = metadata_to_lsn;
6992 xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy
6993 } else if (xtrabackup_prepare && xtrabackup_incremental_dir) {
6994 char filename[FN_REFLEN];
6995
6996 sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME);
6997
6998 if (!xtrabackup_read_metadata(filename)) {
6999 msg("mariabackup: error: failed to read metadata from "
7000 "%s", filename);
7001 return(EXIT_FAILURE);
7002 }
7003
7004 incremental_lsn = metadata_from_lsn;
7005 incremental_to_lsn = metadata_to_lsn;
7006 incremental_last_lsn = metadata_last_lsn;
7007 xtrabackup_incremental = xtrabackup_incremental_dir; //dummy
7008
7009 } else if (opt_incremental_history_name) {
7010 xtrabackup_incremental = opt_incremental_history_name;
7011 } else if (opt_incremental_history_uuid) {
7012 xtrabackup_incremental = opt_incremental_history_uuid;
7013 } else {
7014 xtrabackup_incremental = NULL;
7015 }
7016
7017 if (xtrabackup_stream && !xtrabackup_backup) {
7018 msg("Warning: --stream parameter is ignored, it only works together with --backup.");
7019 }
7020
7021 if (!xb_init()) {
7022 return(EXIT_FAILURE);
7023 }
7024
7025 /* --print-param */
7026 if (xtrabackup_print_param) {
7027 printf("%s", print_param_str.str().c_str());
7028 return(EXIT_SUCCESS);
7029 }
7030
7031 print_version();
7032 if (xtrabackup_incremental) {
7033 msg("incremental backup from " LSN_PF " is enabled.",
7034 incremental_lsn);
7035 }
7036
7037 if (xtrabackup_export && innobase_file_per_table == FALSE) {
7038 msg("mariabackup: auto-enabling --innodb-file-per-table due to "
7039 "the --export option");
7040 innobase_file_per_table = TRUE;
7041 }
7042
7043 /* cannot execute both for now */
7044 {
7045 int num = 0;
7046
7047 if (xtrabackup_backup) num++;
7048 if (xtrabackup_prepare) num++;
7049 if (xtrabackup_copy_back) num++;
7050 if (xtrabackup_move_back) num++;
7051 if (xtrabackup_decrypt_decompress) num++;
7052 if (num != 1) { /* !XOR (for now) */
7053 usage();
7054 return(EXIT_FAILURE);
7055 }
7056 }
7057
7058 /* --backup */
7059 if (xtrabackup_backup && !xtrabackup_backup_func()) {
7060 return(EXIT_FAILURE);
7061 }
7062
7063 /* --prepare */
7064 if (xtrabackup_prepare
7065 && !xtrabackup_prepare_func(argv)) {
7066 return(EXIT_FAILURE);
7067 }
7068
7069 if (xtrabackup_copy_back || xtrabackup_move_back) {
7070 if (!check_if_param_set("datadir")) {
7071 mysql_data_home = get_default_datadir();
7072 }
7073 if (!copy_back())
7074 return(EXIT_FAILURE);
7075 }
7076
7077 if (xtrabackup_decrypt_decompress && !decrypt_decompress()) {
7078 return(EXIT_FAILURE);
7079 }
7080
7081 return(EXIT_SUCCESS);
7082 }
7083
7084
get_exepath(char * buf,size_t size,const char * argv0)7085 static int get_exepath(char *buf, size_t size, const char *argv0)
7086 {
7087 #ifdef _WIN32
7088 DWORD ret = GetModuleFileNameA(NULL, buf, (DWORD)size);
7089 if (ret > 0)
7090 return 0;
7091 #elif defined(__linux__)
7092 ssize_t ret = readlink("/proc/self/exe", buf, size-1);
7093 if(ret > 0)
7094 return 0;
7095 #elif defined(__APPLE__)
7096 size_t ret = proc_pidpath(getpid(), buf, static_cast<uint32_t>(size));
7097 if (ret > 0) {
7098 buf[ret] = 0;
7099 return 0;
7100 }
7101 #endif
7102
7103 return my_realpath(buf, argv0, 0);
7104 }
7105
7106
7107 #if defined (__SANITIZE_ADDRESS__) && defined (__linux__)
7108 /* Avoid LeakSanitizer's false positives. */
__asan_default_options()7109 const char* __asan_default_options()
7110 {
7111 return "detect_leaks=0";
7112 }
7113 #endif
7114