1 /******************************************************
2 MariaBackup: hot backup tool for InnoDB
3 (c) 2009-2017 Percona LLC and/or its affiliates
4 Originally Created 3/3/2009 Yasufumi Kinoshita
5 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
6 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
7 (c) 2017, 2021, MariaDB Corporation.
8 Portions written by Marko Mäkelä.
9
10 This program is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; version 2 of the License.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
22
23 *******************************************************
24
25 This file incorporates work covered by the following copyright and
26 permission notice:
27
28 Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
29
30 This program is free software; you can redistribute it and/or modify it under
31 the terms of the GNU General Public License as published by the Free Software
32 Foundation; version 2 of the License.
33
34 This program is distributed in the hope that it will be useful, but WITHOUT
35 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
36 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
37
38 You should have received a copy of the GNU General Public License along with
39 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
40 Street, Fifth Floor, Boston, MA 02110-1335 USA
41
42 *******************************************************/
43
44 //#define XTRABACKUP_TARGET_IS_PLUGIN
45
46 #include <my_global.h>
47 #include <my_config.h>
48 #include <unireg.h>
49 #include <mysql_version.h>
50 #include <my_base.h>
51 #include <my_getopt.h>
52 #include <mysql_com.h>
53 #include <my_default.h>
54 #include <mysqld.h>
55
56 #include <fcntl.h>
57 #include <string.h>
58
59 #ifdef __linux__
60 # include <sys/prctl.h>
61 #include <sys/resource.h>
62 #endif
63
64
65 #include <btr0sea.h>
66 #include <dict0priv.h>
67 #include <lock0lock.h>
68 #include <log0recv.h>
69 #include <log0crypt.h>
70 #include <row0mysql.h>
71 #include <row0quiesce.h>
72 #include <srv0start.h>
73 #include "trx0sys.h"
74 #include <buf0dblwr.h>
75 #include "ha_innodb.h"
76
77 #include <list>
78 #include <sstream>
79 #include <set>
80 #include <fstream>
81 #include <mysql.h>
82
83 #define G_PTR uchar*
84
85 #include "common.h"
86 #include "datasink.h"
87
88 #include "xb_regex.h"
89 #include "fil_cur.h"
90 #include "write_filt.h"
91 #include "xtrabackup.h"
92 #include "ds_buffer.h"
93 #include "ds_tmpfile.h"
94 #include "xbstream.h"
95 #include "changed_page_bitmap.h"
96 #include "read_filt.h"
97 #include "backup_wsrep.h"
98 #include "innobackupex.h"
99 #include "backup_mysql.h"
100 #include "backup_copy.h"
101 #include "backup_mysql.h"
102 #include "encryption_plugin.h"
103 #include <sql_plugin.h>
104 #include <srv0srv.h>
105 #include <crc_glue.h>
106 #include <log.h>
107 #include <derror.h>
108 #include "backup_debug.h"
109
110 #define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages"
111
112 int sys_var_init();
113
114 /* === xtrabackup specific options === */
115 char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/";
116 char *xtrabackup_target_dir= xtrabackup_real_target_dir;
117 static my_bool xtrabackup_version;
118 static my_bool verbose;
119 my_bool xtrabackup_backup;
120 my_bool xtrabackup_prepare;
121 my_bool xtrabackup_copy_back;
122 my_bool xtrabackup_move_back;
123 my_bool xtrabackup_decrypt_decompress;
124 my_bool xtrabackup_print_param;
125
126 my_bool xtrabackup_export;
127
128 my_bool xtrabackup_rollback_xa;
129
130 longlong xtrabackup_use_memory;
131
132 uint opt_protocol;
133 long xtrabackup_throttle; /* 0:unlimited */
134 static lint io_ticket;
135 static os_event_t wait_throttle;
136 static os_event_t log_copying_stop;
137
138 char *xtrabackup_incremental;
139 lsn_t incremental_lsn;
140 lsn_t incremental_to_lsn;
141 lsn_t incremental_last_lsn;
142 xb_page_bitmap *changed_page_bitmap;
143
144 char *xtrabackup_incremental_basedir; /* for --backup */
145 char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */
146 char *xtrabackup_incremental_dir; /* for --prepare */
147
148 char xtrabackup_real_incremental_basedir[FN_REFLEN];
149 char xtrabackup_real_extra_lsndir[FN_REFLEN];
150 char xtrabackup_real_incremental_dir[FN_REFLEN];
151
152
153 char *xtrabackup_tmpdir;
154
155 char *xtrabackup_tables;
156 char *xtrabackup_tables_file;
157 char *xtrabackup_tables_exclude;
158 char *xb_rocksdb_datadir;
159 my_bool xb_backup_rocksdb = 1;
160
161 typedef std::list<regex_t> regex_list_t;
162 static regex_list_t regex_include_list;
163 static regex_list_t regex_exclude_list;
164
165 static hash_table_t* tables_include_hash = NULL;
166 static hash_table_t* tables_exclude_hash = NULL;
167
168 char *xtrabackup_databases = NULL;
169 char *xtrabackup_databases_file = NULL;
170 char *xtrabackup_databases_exclude = NULL;
171 static hash_table_t* databases_include_hash = NULL;
172 static hash_table_t* databases_exclude_hash = NULL;
173
174 static hash_table_t* inc_dir_tables_hash;
175
176 struct xb_filter_entry_struct{
177 char* name;
178 ibool has_tables;
179 hash_node_t name_hash;
180 };
181 typedef struct xb_filter_entry_struct xb_filter_entry_t;
182
183 lsn_t checkpoint_lsn_start;
184 lsn_t checkpoint_no_start;
185 static lsn_t log_copy_scanned_lsn;
186 static bool log_copying_running;
187 static bool io_watching_thread_running;
188
189 int xtrabackup_parallel;
190
191 char *xtrabackup_stream_str = NULL;
192 xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE;
193 ibool xtrabackup_stream = FALSE;
194
195 const char *xtrabackup_compress_alg = NULL;
196 uint xtrabackup_compress = FALSE;
197 uint xtrabackup_compress_threads;
198 ulonglong xtrabackup_compress_chunk_size = 0;
199
200 /* sleep interval beetween log copy iterations in log copying thread
201 in milliseconds (default is 1 second) */
202 ulint xtrabackup_log_copy_interval = 1000;
203 static ulong max_buf_pool_modified_pct;
204
205 /* Ignored option (--log) for MySQL option compatibility */
206 static char* log_ignored_opt;
207
208
209 extern my_bool opt_use_ssl;
210 my_bool opt_ssl_verify_server_cert;
211 my_bool opt_extended_validation;
212 my_bool opt_encrypted_backup;
213
214 /* === metadata of backup === */
215 #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
216 char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
217 static lsn_t metadata_from_lsn;
218 lsn_t metadata_to_lsn;
219 static lsn_t metadata_last_lsn;
220
221 static ds_file_t* dst_log_file;
222
223 static char mysql_data_home_buff[2];
224
225 const char *defaults_group = "mysqld";
226
227 /* === static parameters in ha_innodb.cc */
228
229 #define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */
230 #define HA_INNOBASE_RANGE_COUNT 100
231
232 /* The default values for the following, type long or longlong, start-up
233 parameters are declared in mysqld.cc: */
234
235 long innobase_buffer_pool_awe_mem_mb = 0;
236 long innobase_file_io_threads = 4;
237 long innobase_read_io_threads = 4;
238 long innobase_write_io_threads = 4;
239
240 longlong innobase_page_size = (1LL << 14); /* 16KB */
241 char* innobase_buffer_pool_filename = NULL;
242
243 /* The default values for the following char* start-up parameters
244 are determined in innobase_init below: */
245
246 static char* innobase_ignored_opt;
247 char* innobase_data_home_dir;
248 char* innobase_data_file_path;
249
250 my_bool innobase_use_doublewrite;
251 my_bool innobase_file_per_table;
252 my_bool innobase_locks_unsafe_for_binlog;
253 my_bool innobase_rollback_on_timeout;
254 my_bool innobase_create_status_file;
255
256 /* The following counter is used to convey information to InnoDB
257 about server activity: in selects it is not sensible to call
258 srv_active_wake_master_thread after each fetch or search, we only do
259 it every INNOBASE_WAKE_INTERVAL'th step. */
260
261 #define INNOBASE_WAKE_INTERVAL 32
262 ulong innobase_active_counter = 0;
263
264
265 my_bool xtrabackup_incremental_force_scan = FALSE;
266
267 /*
268 * Ignore corrupt pages (disabled by default; used
269 * by "innobackupex" as a command line argument).
270 */
271 ulong xtrabackup_innodb_force_recovery = 0;
272
273 /* The flushed lsn which is read from data files */
274 lsn_t flushed_lsn= 0;
275
276 ulong xb_open_files_limit= 0;
277 char *xb_plugin_dir;
278 char *xb_plugin_load;
279 my_bool xb_close_files;
280
281 /* Datasinks */
282 ds_ctxt_t *ds_data = NULL;
283 ds_ctxt_t *ds_meta = NULL;
284 ds_ctxt_t *ds_redo = NULL;
285
286 static bool innobackupex_mode = false;
287
288 /* String buffer used by --print-param to accumulate server options as they are
289 parsed from the defaults file */
290 static std::ostringstream print_param_str;
291
292 /* Set of specified parameters */
293 std::set<std::string> param_set;
294
295 static ulonglong global_max_value;
296
297 extern "C" sig_handler handle_fatal_signal(int sig);
298 extern LOGGER logger;
299
300 my_bool opt_galera_info = FALSE;
301 my_bool opt_slave_info = FALSE;
302 my_bool opt_no_lock = FALSE;
303 my_bool opt_safe_slave_backup = FALSE;
304 my_bool opt_rsync = FALSE;
305 my_bool opt_force_non_empty_dirs = FALSE;
306 my_bool opt_noversioncheck = FALSE;
307 my_bool opt_no_backup_locks = FALSE;
308 my_bool opt_decompress = FALSE;
309 my_bool opt_remove_original;
310 my_bool opt_log_innodb_page_corruption;
311
312 my_bool opt_lock_ddl_per_table = FALSE;
313 static my_bool opt_check_privileges;
314
315 extern const char *innodb_checksum_algorithm_names[];
316 extern TYPELIB innodb_checksum_algorithm_typelib;
317 extern const char *innodb_flush_method_names[];
318 extern TYPELIB innodb_flush_method_typelib;
319
320 static const char *binlog_info_values[] = {"off", "lockless", "on", "auto",
321 NullS};
322 static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "",
323 binlog_info_values, NULL};
324 ulong opt_binlog_info;
325
326 char *opt_incremental_history_name;
327 char *opt_incremental_history_uuid;
328
329 char *opt_user;
330 char *opt_password;
331 char *opt_host;
332 char *opt_defaults_group;
333 char *opt_socket;
334 uint opt_port;
335 char *opt_log_bin;
336
337 const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS};
338
339 TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "",
340 query_type_names, NULL};
341
342 ulong opt_lock_wait_query_type;
343 ulong opt_kill_long_query_type;
344
345 uint opt_kill_long_queries_timeout = 0;
346 uint opt_lock_wait_timeout = 0;
347 uint opt_lock_wait_threshold = 0;
348 uint opt_debug_sleep_before_unlock = 0;
349 uint opt_safe_slave_backup_timeout = 0;
350
351 const char *opt_history = NULL;
352
353
354 char mariabackup_exe[FN_REFLEN];
355 char orig_argv1[FN_REFLEN];
356
357 pthread_mutex_t backup_mutex;
358 pthread_cond_t scanned_lsn_cond;
359
360 typedef std::map<space_id_t,std::string> space_id_to_name_t;
361
362 struct ddl_tracker_t {
363 /** Tablspaces with their ID and name, as they were copied to backup.*/
364 space_id_to_name_t tables_in_backup;
365 /** Tablespaces for that optimized DDL without redo log was found.*/
366 std::set<space_id_t> optimized_ddl;
367 /** Drop operations found in redo log. */
368 std::set<space_id_t> drops;
369 /* For DDL operation found in redo log, */
370 space_id_to_name_t id_to_name;
371 };
372
373 static ddl_tracker_t ddl_tracker;
374
375 // Convert non-null terminated filename to space name
376 std::string filename_to_spacename(const byte *filename, size_t len);
377
CorruptedPages()378 CorruptedPages::CorruptedPages() { ut_a(!pthread_mutex_init(&m_mutex, NULL)); }
379
~CorruptedPages()380 CorruptedPages::~CorruptedPages() { ut_a(!pthread_mutex_destroy(&m_mutex)); }
381
add_page_no_lock(const char * space_name,ulint space_id,ulint page_no,bool convert_space_name)382 void CorruptedPages::add_page_no_lock(const char *space_name, ulint space_id,
383 ulint page_no, bool convert_space_name)
384 {
385 space_info_t &space_info = m_spaces[space_id];
386 if (space_info.space_name.empty())
387 space_info.space_name=
388 convert_space_name
389 ? filename_to_spacename(reinterpret_cast<const byte *>(space_name),
390 strlen(space_name))
391 : space_name;
392 (void)space_info.pages.insert(page_no);
393 }
394
add_page(const char * file_name,ulint space_id,ulint page_no)395 void CorruptedPages::add_page(const char *file_name, ulint space_id,
396 ulint page_no)
397 {
398 ut_a(!pthread_mutex_lock(&m_mutex));
399 add_page_no_lock(file_name, space_id, page_no, true);
400 ut_a(!pthread_mutex_unlock(&m_mutex));
401 }
402
contains(ulint space_id,ulint page_no) const403 bool CorruptedPages::contains(ulint space_id, ulint page_no) const
404 {
405 bool result = false;
406 ut_a(!pthread_mutex_lock(&m_mutex));
407 container_t::const_iterator space_it= m_spaces.find(space_id);
408 if (space_it != m_spaces.end())
409 result = space_it->second.pages.count(page_no);
410 ut_a(!pthread_mutex_unlock(&m_mutex));
411 return result;
412 }
413
drop_space(ulint space_id)414 void CorruptedPages::drop_space(ulint space_id)
415 {
416 ut_a(!pthread_mutex_lock(&m_mutex));
417 m_spaces.erase(space_id);
418 ut_a(!pthread_mutex_unlock(&m_mutex));
419 }
420
rename_space(ulint space_id,const std::string & new_name)421 void CorruptedPages::rename_space(ulint space_id, const std::string &new_name)
422 {
423 ut_a(!pthread_mutex_lock(&m_mutex));
424 container_t::iterator space_it = m_spaces.find(space_id);
425 if (space_it != m_spaces.end())
426 space_it->second.space_name = new_name;
427 ut_a(!pthread_mutex_unlock(&m_mutex));
428 }
429
print_to_file(const char * filename) const430 bool CorruptedPages::print_to_file(const char *filename) const
431 {
432 std::ostringstream out;
433 ut_a(!pthread_mutex_lock(&m_mutex));
434 if (!m_spaces.size())
435 {
436 ut_a(!pthread_mutex_unlock(&m_mutex));
437 return true;
438 }
439 for (container_t::const_iterator space_it=
440 m_spaces.begin();
441 space_it != m_spaces.end(); ++space_it)
442 {
443 out << space_it->second.space_name << " " << space_it->first << "\n";
444 bool first_page_no= true;
445 for (std::set<ulint>::const_iterator page_it=
446 space_it->second.pages.begin();
447 page_it != space_it->second.pages.end(); ++page_it)
448 if (first_page_no)
449 {
450 out << *page_it;
451 first_page_no= false;
452 }
453 else
454 out << " " << *page_it;
455 out << "\n";
456 }
457 ut_a(!pthread_mutex_unlock(&m_mutex));
458 if (xtrabackup_backup)
459 return backup_file_print_buf(filename, out.str().c_str(),
460 static_cast<int>(out.str().size()));
461 std::ofstream outfile;
462 outfile.open(filename);
463 if (!outfile.is_open())
464 die("Can't open %s, error number: %d, error message: %s", filename, errno,
465 strerror(errno));
466 outfile << out.str();
467 return true;
468 }
469
read_from_file(const char * file_name)470 void CorruptedPages::read_from_file(const char *file_name)
471 {
472 MY_STAT mystat;
473 if (!my_stat(file_name, &mystat, MYF(0)))
474 return;
475 std::ifstream infile;
476 infile.open(file_name);
477 if (!infile.is_open())
478 die("Can't open %s, error number: %d, error message: %s", file_name, errno,
479 strerror(errno));
480 std::string line;
481 std::string space_name;
482 ulint space_id;
483 ulint line_number= 0;
484 while (std::getline(infile, line))
485 {
486 ++line_number;
487 std::istringstream iss(line);
488 if (line_number & 1) {
489 if (!(iss >> space_name))
490 die("Can't parse space name from corrupted pages file at "
491 "line " ULINTPF,
492 line_number);
493 if (!(iss >> space_id))
494 die("Can't parse space id from corrupted pages file at line " ULINTPF,
495 line_number);
496 }
497 else
498 {
499 ulint page_no;
500 while ((iss >> page_no))
501 add_page_no_lock(space_name.c_str(), space_id, page_no, false);
502 if (!iss.eof())
503 die("Corrupted pages file parse error on line number " ULINTPF,
504 line_number);
505 }
506 }
507 }
508
empty() const509 bool CorruptedPages::empty() const
510 {
511 ut_a(!pthread_mutex_lock(&m_mutex));
512 bool result= !m_spaces.size();
513 ut_a(!pthread_mutex_unlock(&m_mutex));
514 return result;
515 }
516
517 static void xb_load_single_table_tablespace(const std::string &space_name,
518 bool set_size);
519 static void xb_data_files_close();
520 static fil_space_t* fil_space_get_by_name(const char* name);
521
zero_out_free_pages()522 void CorruptedPages::zero_out_free_pages()
523 {
524 container_t non_free_pages;
525 byte* buf= static_cast<byte*>(ut_malloc_nokey(2 * srv_page_size));
526 byte* zero_page = static_cast<byte*>(ut_align(buf, srv_page_size));
527 memset(zero_page, 0, srv_page_size);
528
529 ut_a(!pthread_mutex_lock(&m_mutex));
530 for (container_t::const_iterator space_it= m_spaces.begin();
531 space_it != m_spaces.end(); ++space_it)
532 {
533 ulint space_id = space_it->first;
534 const std::string &space_name = space_it->second.space_name;
535 // There is no need to close tablespaces explixitly as they will be closed
536 // in innodb_shutdown().
537 xb_load_single_table_tablespace(space_name, false);
538 mutex_enter(&fil_system.mutex);
539 fil_space_t *space = fil_space_get_by_name(space_name.c_str());
540 mutex_exit(&fil_system.mutex);
541 if (!space)
542 die("Can't find space object for space name %s to check corrupted page",
543 space_name.c_str());
544 for (std::set<ulint>::const_iterator page_it=
545 space_it->second.pages.begin();
546 page_it != space_it->second.pages.end(); ++page_it)
547 {
548 bool is_free= fseg_page_is_free(space, static_cast<unsigned>(*page_it));
549 if (!is_free) {
550 space_info_t &space_info = non_free_pages[space_id];
551 space_info.pages.insert(*page_it);
552 if (space_info.space_name.empty())
553 space_info.space_name = space_name;
554 msg("Error: corrupted page " ULINTPF
555 " of tablespace %s can not be fixed",
556 *page_it, space_name.c_str());
557 }
558 else
559 {
560 const page_id_t page_id(space->id, *page_it);
561 dberr_t err= fil_io(IORequestWrite, true, page_id, univ_page_size, 0,
562 univ_page_size.physical(), zero_page, NULL);
563 if (err != DB_SUCCESS)
564 die("Can't zero out corrupted page " ULINTPF " of tablespace %s",
565 *page_it, space_name.c_str());
566 msg("Corrupted page " ULINTPF
567 " of tablespace %s was successfuly fixed.",
568 *page_it, space_name.c_str());
569 }
570 }
571 }
572 m_spaces.swap(non_free_pages);
573 ut_a(!pthread_mutex_unlock(&m_mutex));
574 ut_free(buf);
575 }
576
577 /* Simple datasink creation tracking...add datasinks in the reverse order you
578 want them destroyed. */
579 #define XTRABACKUP_MAX_DATASINKS 10
580 static ds_ctxt_t *datasinks[XTRABACKUP_MAX_DATASINKS];
581 static uint actual_datasinks = 0;
582 static inline
583 void
xtrabackup_add_datasink(ds_ctxt_t * ds)584 xtrabackup_add_datasink(ds_ctxt_t *ds)
585 {
586 xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS);
587 datasinks[actual_datasinks] = ds; actual_datasinks++;
588 }
589
590 typedef void (*process_single_tablespace_func_t)(const char *dirname,
591 const char *filname,
592 bool is_remote,
593 bool set_size);
594 static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback);
595
596 /* ======== Datafiles iterator ======== */
597 struct datafiles_iter_t {
598 fil_space_t *space;
599 fil_node_t *node;
600 ibool started;
601 pthread_mutex_t mutex;
602 };
603
604 /* ======== Datafiles iterator ======== */
605 static
606 datafiles_iter_t *
datafiles_iter_new()607 datafiles_iter_new()
608 {
609 datafiles_iter_t *it;
610
611 it = static_cast<datafiles_iter_t *>(malloc(sizeof(datafiles_iter_t)));
612 pthread_mutex_init(&it->mutex, NULL);
613
614 it->space = NULL;
615 it->node = NULL;
616 it->started = FALSE;
617
618 return it;
619 }
620
621 static
622 fil_node_t *
datafiles_iter_next(datafiles_iter_t * it)623 datafiles_iter_next(datafiles_iter_t *it)
624 {
625 fil_node_t *new_node;
626
627 pthread_mutex_lock(&it->mutex);
628
629 if (it->node == NULL) {
630 if (it->started)
631 goto end;
632 it->started = TRUE;
633 } else {
634 it->node = UT_LIST_GET_NEXT(chain, it->node);
635 if (it->node != NULL)
636 goto end;
637 }
638
639 it->space = (it->space == NULL) ?
640 UT_LIST_GET_FIRST(fil_system.space_list) :
641 UT_LIST_GET_NEXT(space_list, it->space);
642
643 while (it->space != NULL &&
644 (it->space->purpose != FIL_TYPE_TABLESPACE ||
645 UT_LIST_GET_LEN(it->space->chain) == 0))
646 it->space = UT_LIST_GET_NEXT(space_list, it->space);
647 if (it->space == NULL)
648 goto end;
649
650 it->node = UT_LIST_GET_FIRST(it->space->chain);
651
652 end:
653 new_node = it->node;
654 pthread_mutex_unlock(&it->mutex);
655
656 return new_node;
657 }
658
659 static
660 void
datafiles_iter_free(datafiles_iter_t * it)661 datafiles_iter_free(datafiles_iter_t *it)
662 {
663 pthread_mutex_destroy(&it->mutex);
664 free(it);
665 }
666
667 #ifndef DBUG_OFF
668 struct dbug_thread_param_t
669 {
670 MYSQL *con;
671 const char *query;
672 int expect_err;
673 int expect_errno;
674 os_event_t done_event;
675 };
676
677
678 /* Thread procedure used in dbug_start_query_thread. */
679 extern "C"
680 os_thread_ret_t
DECLARE_THREAD(dbug_execute_in_new_connection)681 DECLARE_THREAD(dbug_execute_in_new_connection)(void *arg)
682 {
683 mysql_thread_init();
684 dbug_thread_param_t *par= (dbug_thread_param_t *)arg;
685 int err = mysql_query(par->con, par->query);
686 int err_no = mysql_errno(par->con);
687 DBUG_ASSERT(par->expect_err == err);
688 if (err && par->expect_errno)
689 DBUG_ASSERT(err_no == par->expect_errno);
690 mysql_close(par->con);
691 mysql_thread_end();
692 os_event_t done = par->done_event;
693 delete par;
694 os_event_set(done);
695 os_thread_exit();
696 return os_thread_ret_t(0);
697 }
698
699 /*
700 Execute query from a new connection, in own thread.
701
702 @param query - query to be executed
703 @param wait_state - if not NULL, wait until query from new connection
704 reaches this state (value of column State in I_S.PROCESSLIST)
705 @param expected_err - if 0, query is supposed to finish successfully,
706 otherwise query should return error.
707 @param expected_errno - if not 0, and query finished with error,
708 expected mysql_errno()
709 */
dbug_start_query_thread(const char * query,const char * wait_state,int expected_err,int expected_errno)710 static os_event_t dbug_start_query_thread(
711 const char *query,
712 const char *wait_state,
713 int expected_err,
714 int expected_errno)
715
716 {
717 dbug_thread_param_t *par = new dbug_thread_param_t;
718 par->query = query;
719 par->expect_err = expected_err;
720 par->expect_errno = expected_errno;
721 par->done_event = os_event_create(0);
722 par->con = xb_mysql_connect();
723 os_thread_create(dbug_execute_in_new_connection, par, 0);
724
725 if (!wait_state)
726 return par->done_event;
727
728 char q[256];
729 snprintf(q, sizeof(q),
730 "SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST where ID=%lu"
731 " AND Command='Query' AND State='%s'",
732 mysql_thread_id(par->con), wait_state);
733 for (;;) {
734 MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true);
735 bool exists = mysql_fetch_row(result) != NULL;
736 mysql_free_result(result);
737 if (exists) {
738 goto end;
739 }
740 msg("Waiting for query '%s' on connection %lu to "
741 " reach state '%s'", query, mysql_thread_id(par->con),
742 wait_state);
743 my_sleep(1000);
744 }
745 end:
746 msg("query '%s' on connection %lu reached state '%s'", query,
747 mysql_thread_id(par->con), wait_state);
748 return par->done_event;
749 }
750
751 os_event_t dbug_alter_thread_done;
752 #endif
753
mdl_lock_all()754 void mdl_lock_all()
755 {
756 mdl_lock_init();
757 datafiles_iter_t *it = datafiles_iter_new();
758 if (!it)
759 return;
760
761 while (fil_node_t *node = datafiles_iter_next(it)){
762 if (fil_is_user_tablespace_id(node->space->id)
763 && check_if_skip_table(node->space->name))
764 continue;
765
766 mdl_lock_table(node->space->id);
767 }
768 datafiles_iter_free(it);
769 }
770
771
772 // Convert non-null terminated filename to space name
filename_to_spacename(const byte * filename,size_t len)773 std::string filename_to_spacename(const byte *filename, size_t len)
774 {
775 // null- terminate filename
776 char *f = (char *)malloc(len + 1);
777 ut_a(f);
778 memcpy(f, filename, len);
779 f[len] = 0;
780 for (size_t i = 0; i < len; i++)
781 if (f[i] == '\\')
782 f[i] = '/';
783 char *p = strrchr(f, '.');
784 ut_a(p);
785 *p = 0;
786 char *table = strrchr(f, '/');
787 ut_a(table);
788 *table = 0;
789 char *db = strrchr(f, '/');
790 ut_a(db);
791 *table = '/';
792 std::string s(db+1);
793 free(f);
794 return s;
795 }
796
797 /** Report an operation to create, delete, or rename a file during backup.
798 @param[in] space_id tablespace identifier
799 @param[in] flags tablespace flags (NULL if not create)
800 @param[in] name file name (not NUL-terminated)
801 @param[in] len length of name, in bytes
802 @param[in] new_name new file name (NULL if not rename)
803 @param[in] new_len length of new_name, in bytes (0 if NULL) */
backup_file_op(ulint space_id,const byte * flags,const byte * name,ulint len,const byte * new_name,ulint new_len)804 static void backup_file_op(ulint space_id, const byte* flags,
805 const byte* name, ulint len,
806 const byte* new_name, ulint new_len)
807 {
808
809 ut_ad(!flags || !new_name);
810 ut_ad(name);
811 ut_ad(len);
812 ut_ad(!new_name == !new_len);
813 pthread_mutex_lock(&backup_mutex);
814
815 if (flags) {
816 ddl_tracker.id_to_name[space_id] = filename_to_spacename(name, len);
817 msg("DDL tracking : create %zu \"%.*s\": %x",
818 space_id, int(len), name, mach_read_from_4(flags));
819 }
820 else if (new_name) {
821 ddl_tracker.id_to_name[space_id] = filename_to_spacename(new_name, new_len);
822 msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"",
823 space_id, int(len), name, int(new_len), new_name);
824 } else {
825 ddl_tracker.drops.insert(space_id);
826 msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name);
827 }
828 pthread_mutex_unlock(&backup_mutex);
829 }
830
831
832 /*
833 This callback is called if DDL operation is detected,
834 at the end of backup
835
836 Normally, DDL operations are blocked due to FTWRL,
837 but in rare cases of --no-lock, they are not.
838
839 We will abort backup in this case.
840 */
backup_file_op_fail(ulint space_id,const byte * flags,const byte * name,ulint len,const byte * new_name,ulint new_len)841 static void backup_file_op_fail(ulint space_id, const byte* flags,
842 const byte* name, ulint len,
843 const byte* new_name, ulint new_len)
844 {
845 ut_a(opt_no_lock);
846 bool fail;
847 if (flags) {
848 msg("DDL tracking : create %zu \"%.*s\": %x",
849 space_id, int(len), name, mach_read_from_4(flags));
850 std::string spacename = filename_to_spacename(name, len);
851 fail = !check_if_skip_table(spacename.c_str());
852 }
853 else if (new_name) {
854 msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"",
855 space_id, int(len), name, int(new_len), new_name);
856 std::string spacename = filename_to_spacename(name, len);
857 std::string new_spacename = filename_to_spacename(new_name, new_len);
858 fail = !check_if_skip_table(spacename.c_str()) || !check_if_skip_table(new_spacename.c_str());
859 }
860 else {
861 std::string spacename = filename_to_spacename(name, len);
862 fail = !check_if_skip_table(spacename.c_str());
863 msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name);
864 }
865 if (fail) {
866 die("DDL operation detected in the late phase of backup."
867 "Backup is inconsistent. Remove --no-lock option to fix.");
868 }
869 }
870
871
872 /** Callback whenever MLOG_INDEX_LOAD happens.
873 @param[in] space_id space id to check */
backup_optimized_ddl_op(ulint space_id)874 static void backup_optimized_ddl_op(ulint space_id)
875 {
876 pthread_mutex_lock(&backup_mutex);
877 ddl_tracker.optimized_ddl.insert(space_id);
878 pthread_mutex_unlock(&backup_mutex);
879 }
880
881 /*
882 Optimized DDL callback at the end of backup that
883 run with --no-lock. Usually aborts the backup.
884 */
backup_optimized_ddl_op_fail(ulint space_id)885 static void backup_optimized_ddl_op_fail(ulint space_id) {
886 ut_a(opt_no_lock);
887 msg("DDL tracking : optimized DDL on space %zu", space_id);
888 if (ddl_tracker.tables_in_backup.find(space_id) != ddl_tracker.tables_in_backup.end()) {
889 msg("ERROR : Optimized DDL operation detected in the late phase of backup."
890 "Backup is inconsistent. Remove --no-lock option to fix.");
891 exit(EXIT_FAILURE);
892 }
893 }
894
895
896 /** Callback whenever MLOG_TRUNCATE happens. */
backup_truncate_fail()897 static void backup_truncate_fail()
898 {
899 msg("mariabackup: Incompatible TRUNCATE operation detected.%s",
900 opt_lock_ddl_per_table
901 ? ""
902 : " Use --lock-ddl-per-table to lock all tables before backup.");
903 }
904
905
906 /*
907 Retrieve default data directory, to be used with --copy-back.
908
909 On Windows, default datadir is ..\data, relative to the
910 directory where mariabackup.exe is located(usually "bin")
911
912 Elsewhere, the compiled-in constant MYSQL_DATADIR is used.
913 */
get_default_datadir()914 static char *get_default_datadir() {
915 static char ddir[] = MYSQL_DATADIR;
916 #ifdef _WIN32
917 static char buf[MAX_PATH];
918 DWORD size = (DWORD)sizeof(buf) - 1;
919 if (GetModuleFileName(NULL, buf, size) <= size)
920 {
921 char *p;
922 if ((p = strrchr(buf, '\\')))
923 {
924 *p = 0;
925 if ((p = strrchr(buf, '\\')))
926 {
927 strncpy(p + 1, "data", buf + MAX_PATH - p);
928 return buf;
929 }
930 }
931 }
932 #endif
933 return ddir;
934 }
935
936
937 /* ======== Date copying thread context ======== */
938
939 typedef struct {
940 datafiles_iter_t *it;
941 uint num;
942 uint *count;
943 pthread_mutex_t* count_mutex;
944 os_thread_id_t id;
945 CorruptedPages *corrupted_pages;
946 } data_thread_ctxt_t;
947
948 /* ======== for option and variables ======== */
949 #include <../../client/client_priv.h>
950
951 enum options_xtrabackup
952 {
953 OPT_XTRA_TARGET_DIR= 1000, /* make sure it is larger
954 than OPT_MAX_CLIENT_OPTION */
955 OPT_XTRA_BACKUP,
956 OPT_XTRA_PREPARE,
957 OPT_XTRA_EXPORT,
958 OPT_XTRA_ROLLBACK_XA,
959 OPT_XTRA_PRINT_PARAM,
960 OPT_XTRA_USE_MEMORY,
961 OPT_XTRA_THROTTLE,
962 OPT_XTRA_LOG_COPY_INTERVAL,
963 OPT_XTRA_INCREMENTAL,
964 OPT_XTRA_INCREMENTAL_BASEDIR,
965 OPT_XTRA_EXTRA_LSNDIR,
966 OPT_XTRA_INCREMENTAL_DIR,
967 OPT_XTRA_TABLES,
968 OPT_XTRA_TABLES_FILE,
969 OPT_XTRA_DATABASES,
970 OPT_XTRA_DATABASES_FILE,
971 OPT_XTRA_PARALLEL,
972 OPT_XTRA_EXTENDED_VALIDATION,
973 OPT_XTRA_ENCRYPTED_BACKUP,
974 OPT_XTRA_STREAM,
975 OPT_XTRA_COMPRESS,
976 OPT_XTRA_COMPRESS_THREADS,
977 OPT_XTRA_COMPRESS_CHUNK_SIZE,
978 OPT_LOG,
979 OPT_INNODB,
980 OPT_INNODB_DATA_FILE_PATH,
981 OPT_INNODB_DATA_HOME_DIR,
982 OPT_INNODB_ADAPTIVE_HASH_INDEX,
983 OPT_INNODB_DOUBLEWRITE,
984 OPT_INNODB_FILE_PER_TABLE,
985 OPT_INNODB_FLUSH_METHOD,
986 OPT_INNODB_LOG_GROUP_HOME_DIR,
987 OPT_INNODB_MAX_DIRTY_PAGES_PCT,
988 OPT_INNODB_MAX_PURGE_LAG,
989 OPT_INNODB_STATUS_FILE,
990 OPT_INNODB_AUTOEXTEND_INCREMENT,
991 OPT_INNODB_BUFFER_POOL_SIZE,
992 OPT_INNODB_COMMIT_CONCURRENCY,
993 OPT_INNODB_CONCURRENCY_TICKETS,
994 OPT_INNODB_FILE_IO_THREADS,
995 OPT_INNODB_IO_CAPACITY,
996 OPT_INNODB_READ_IO_THREADS,
997 OPT_INNODB_WRITE_IO_THREADS,
998 OPT_INNODB_USE_NATIVE_AIO,
999 OPT_INNODB_PAGE_SIZE,
1000 OPT_INNODB_BUFFER_POOL_FILENAME,
1001 OPT_INNODB_LOCK_WAIT_TIMEOUT,
1002 OPT_INNODB_LOG_BUFFER_SIZE,
1003 OPT_INNODB_LOG_FILE_SIZE,
1004 OPT_INNODB_LOG_FILES_IN_GROUP,
1005 OPT_INNODB_OPEN_FILES,
1006 OPT_XTRA_DEBUG_SYNC,
1007 OPT_INNODB_CHECKSUM_ALGORITHM,
1008 OPT_INNODB_UNDO_DIRECTORY,
1009 OPT_INNODB_UNDO_TABLESPACES,
1010 OPT_INNODB_LOG_CHECKSUMS,
1011 OPT_XTRA_INCREMENTAL_FORCE_SCAN,
1012 OPT_DEFAULTS_GROUP,
1013 OPT_CLOSE_FILES,
1014 OPT_CORE_FILE,
1015
1016 OPT_COPY_BACK,
1017 OPT_MOVE_BACK,
1018 OPT_GALERA_INFO,
1019 OPT_SLAVE_INFO,
1020 OPT_NO_LOCK,
1021 OPT_SAFE_SLAVE_BACKUP,
1022 OPT_RSYNC,
1023 OPT_FORCE_NON_EMPTY_DIRS,
1024 OPT_NO_VERSION_CHECK,
1025 OPT_NO_BACKUP_LOCKS,
1026 OPT_DECOMPRESS,
1027 OPT_INCREMENTAL_HISTORY_NAME,
1028 OPT_INCREMENTAL_HISTORY_UUID,
1029 OPT_REMOVE_ORIGINAL,
1030 OPT_LOCK_WAIT_QUERY_TYPE,
1031 OPT_KILL_LONG_QUERY_TYPE,
1032 OPT_HISTORY,
1033 OPT_KILL_LONG_QUERIES_TIMEOUT,
1034 OPT_LOCK_WAIT_TIMEOUT,
1035 OPT_LOCK_WAIT_THRESHOLD,
1036 OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
1037 OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
1038 OPT_BINLOG_INFO,
1039 OPT_XB_SECURE_AUTH,
1040
1041 OPT_XTRA_TABLES_EXCLUDE,
1042 OPT_XTRA_DATABASES_EXCLUDE,
1043 OPT_PROTOCOL,
1044 OPT_INNODB_COMPRESSION_LEVEL,
1045 OPT_LOCK_DDL_PER_TABLE,
1046 OPT_ROCKSDB_DATADIR,
1047 OPT_BACKUP_ROCKSDB,
1048 OPT_XTRA_CHECK_PRIVILEGES,
1049 OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
1050 OPT_INNODB_FORCE_RECOVERY
1051 };
1052
1053 struct my_option xb_client_options[]= {
1054 {"verbose", 'V', "display verbose output", (G_PTR *) &verbose,
1055 (G_PTR *) &verbose, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
1056 {"version", 'v', "print version information",
1057 (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL,
1058 NO_ARG, 0, 0, 0, 0, 0, 0},
1059 {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory",
1060 (G_PTR *) &xtrabackup_target_dir, (G_PTR *) &xtrabackup_target_dir, 0,
1061 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1062 {"backup", OPT_XTRA_BACKUP, "take backup to target-dir",
1063 (G_PTR *) &xtrabackup_backup, (G_PTR *) &xtrabackup_backup, 0, GET_BOOL,
1064 NO_ARG, 0, 0, 0, 0, 0, 0},
1065 {"prepare", OPT_XTRA_PREPARE,
1066 "prepare a backup for starting mysql server on the backup.",
1067 (G_PTR *) &xtrabackup_prepare, (G_PTR *) &xtrabackup_prepare, 0, GET_BOOL,
1068 NO_ARG, 0, 0, 0, 0, 0, 0},
1069 {"export", OPT_XTRA_EXPORT,
1070 "create files to import to another database when prepare.",
1071 (G_PTR *) &xtrabackup_export, (G_PTR *) &xtrabackup_export, 0, GET_BOOL,
1072 NO_ARG, 0, 0, 0, 0, 0, 0},
1073 {"rollback-xa", OPT_XTRA_ROLLBACK_XA,
1074 "Rollback prepared XA's on --prepare. "
1075 "After preparing target directory with this option "
1076 "it can no longer be a base for incremental backup.",
1077 (G_PTR *) &xtrabackup_rollback_xa, (G_PTR *) &xtrabackup_rollback_xa, 0,
1078 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1079 {"print-param", OPT_XTRA_PRINT_PARAM,
1080 "print parameter of mysqld needed for copyback.",
1081 (G_PTR *) &xtrabackup_print_param, (G_PTR *) &xtrabackup_print_param, 0,
1082 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1083 {"use-memory", OPT_XTRA_USE_MEMORY,
1084 "The value is used instead of buffer_pool_size",
1085 (G_PTR *) &xtrabackup_use_memory, (G_PTR *) &xtrabackup_use_memory, 0,
1086 GET_LL, REQUIRED_ARG, 100 * 1024 * 1024L, 1024 * 1024L, LONGLONG_MAX, 0,
1087 1024 * 1024L, 0},
1088 {"throttle", OPT_XTRA_THROTTLE,
1089 "limit count of IO operations (pairs of read&write) per second to IOS "
1090 "values (for '--backup')",
1091 (G_PTR *) &xtrabackup_throttle, (G_PTR *) &xtrabackup_throttle, 0,
1092 GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0},
1093 {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
1094 (G_PTR *) &log_ignored_opt, (G_PTR *) &log_ignored_opt, 0, GET_STR,
1095 OPT_ARG, 0, 0, 0, 0, 0, 0},
1096 {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL,
1097 "time interval between checks done by log copying thread in milliseconds "
1098 "(default is 1 second).",
1099 (G_PTR *) &xtrabackup_log_copy_interval,
1100 (G_PTR *) &xtrabackup_log_copy_interval, 0, GET_LONG, REQUIRED_ARG, 1000,
1101 0, LONG_MAX, 0, 1, 0},
1102 {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR,
1103 "(for --backup): save an extra copy of the xtrabackup_checkpoints file "
1104 "in this directory.",
1105 (G_PTR *) &xtrabackup_extra_lsndir, (G_PTR *) &xtrabackup_extra_lsndir, 0,
1106 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1107 {"incremental-lsn", OPT_XTRA_INCREMENTAL,
1108 "(for --backup): copy only .ibd pages newer than specified LSN "
1109 "'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is "
1110 "impossible to diagnose this, causing the backup to be unusable. Be "
1111 "careful!",
1112 (G_PTR *) &xtrabackup_incremental, (G_PTR *) &xtrabackup_incremental, 0,
1113 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1114 {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR,
1115 "(for --backup): copy only .ibd pages newer than backup at specified "
1116 "directory.",
1117 (G_PTR *) &xtrabackup_incremental_basedir,
1118 (G_PTR *) &xtrabackup_incremental_basedir, 0, GET_STR, REQUIRED_ARG, 0, 0,
1119 0, 0, 0, 0},
1120 {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR,
1121 "(for --prepare): apply .delta files and logfile in the specified "
1122 "directory.",
1123 (G_PTR *) &xtrabackup_incremental_dir,
1124 (G_PTR *) &xtrabackup_incremental_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
1125 0, 0, 0},
1126 {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.",
1127 (G_PTR *) &xtrabackup_tables, (G_PTR *) &xtrabackup_tables, 0, GET_STR,
1128 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1129 {"tables_file", OPT_XTRA_TABLES_FILE,
1130 "filtering by list of the exact database.table name in the file.",
1131 (G_PTR *) &xtrabackup_tables_file, (G_PTR *) &xtrabackup_tables_file, 0,
1132 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1133 {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.",
1134 (G_PTR *) &xtrabackup_databases, (G_PTR *) &xtrabackup_databases, 0,
1135 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1136 {"databases_file", OPT_XTRA_DATABASES_FILE,
1137 "filtering by list of databases in the file.",
1138 (G_PTR *) &xtrabackup_databases_file,
1139 (G_PTR *) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
1140 0, 0, 0},
1141 {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE,
1142 "filtering by regexp for table names. "
1143 "Operates the same way as --tables, but matched names are excluded from "
1144 "backup. "
1145 "Note that this option has a higher priority than --tables.",
1146 (G_PTR *) &xtrabackup_tables_exclude,
1147 (G_PTR *) &xtrabackup_tables_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
1148 0, 0, 0},
1149 {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE,
1150 "Excluding databases based on name, "
1151 "Operates the same way as --databases, but matched names are excluded "
1152 "from backup. "
1153 "Note that this option has a higher priority than --databases.",
1154 (G_PTR *) &xtrabackup_databases_exclude,
1155 (G_PTR *) &xtrabackup_databases_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0,
1156 0, 0, 0, 0},
1157
1158 {"stream", OPT_XTRA_STREAM,
1159 "Stream all backup files to the standard output "
1160 "in the specified format."
1161 "Supported format is 'mbstream' or 'xbstream'.",
1162 (G_PTR *) &xtrabackup_stream_str, (G_PTR *) &xtrabackup_stream_str, 0,
1163 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1164
1165 {"compress", OPT_XTRA_COMPRESS,
1166 "Compress individual backup files using the "
1167 "specified compression algorithm. Currently the only supported algorithm "
1168 "is 'quicklz'. It is also the default algorithm, i.e. the one used when "
1169 "--compress is used without an argument.",
1170 (G_PTR *) &xtrabackup_compress_alg, (G_PTR *) &xtrabackup_compress_alg, 0,
1171 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1172
1173 {"compress-threads", OPT_XTRA_COMPRESS_THREADS,
1174 "Number of threads for parallel data compression. The default value is "
1175 "1.",
1176 (G_PTR *) &xtrabackup_compress_threads,
1177 (G_PTR *) &xtrabackup_compress_threads, 0, GET_UINT, REQUIRED_ARG, 1, 1,
1178 UINT_MAX, 0, 0, 0},
1179
1180 {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE,
1181 "Size of working buffer(s) for compression threads in bytes. The default "
1182 "value is 64K.",
1183 (G_PTR *) &xtrabackup_compress_chunk_size,
1184 (G_PTR *) &xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG,
1185 (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
1186
1187 {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN,
1188 "Perform a full-scan incremental backup even in the presence of changed "
1189 "page bitmap data",
1190 (G_PTR *) &xtrabackup_incremental_force_scan,
1191 (G_PTR *) &xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, 0, 0,
1192 0, 0, 0, 0},
1193
1194 {"close_files", OPT_CLOSE_FILES,
1195 "do not keep files opened. Use at your own "
1196 "risk.",
1197 (G_PTR *) &xb_close_files, (G_PTR *) &xb_close_files, 0, GET_BOOL, NO_ARG,
1198 0, 0, 0, 0, 0, 0},
1199
1200 {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0,
1201 GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
1202
1203 {"copy-back", OPT_COPY_BACK,
1204 "Copy all the files in a previously made "
1205 "backup from the backup directory to their original locations.",
1206 (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0,
1207 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1208
1209 {"move-back", OPT_MOVE_BACK,
1210 "Move all the files in a previously made "
1211 "backup from the backup directory to the actual datadir location. "
1212 "Use with caution, as it removes backup files.",
1213 (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0,
1214 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1215
1216 {"galera-info", OPT_GALERA_INFO,
1217 "This options creates the "
1218 "xtrabackup_galera_info file which contains the local node state at "
1219 "the time of the backup. Option should be used when performing the "
1220 "backup of MariaDB Galera Cluster. Has no effect when backup locks "
1221 "are used to create the backup.",
1222 (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0, GET_BOOL,
1223 NO_ARG, 0, 0, 0, 0, 0, 0},
1224
1225 {"slave-info", OPT_SLAVE_INFO,
1226 "This option is useful when backing "
1227 "up a replication slave server. It prints the binary log position "
1228 "and name of the master server. It also writes this information to "
1229 "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. "
1230 "A new slave for this master can be set up by starting a slave server "
1231 "on this backup and issuing a \"CHANGE MASTER\" command with the "
1232 "binary log position saved in the \"xtrabackup_slave_info\" file.",
1233 (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0, GET_BOOL, NO_ARG,
1234 0, 0, 0, 0, 0, 0},
1235
1236 {"no-lock", OPT_NO_LOCK,
1237 "Use this option to disable table lock "
1238 "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
1239 "tables are InnoDB and you DO NOT CARE about the binary log "
1240 "position of the backup. This option shouldn't be used if there "
1241 "are any DDL statements being executed or if any updates are "
1242 "happening on non-InnoDB tables (this includes the system MyISAM "
1243 "tables in the mysql database), otherwise it could lead to an "
1244 "inconsistent backup. If you are considering to use --no-lock "
1245 "because your backups are failing to acquire the lock, this could "
1246 "be because of incoming replication events preventing the lock "
1247 "from succeeding. Please try using --safe-slave-backup to "
1248 "momentarily stop the replication slave thread, this may help "
1249 "the backup to succeed and you then don't need to resort to "
1250 "using this option.",
1251 (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0, GET_BOOL, NO_ARG, 0, 0,
1252 0, 0, 0, 0},
1253
1254 {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP,
1255 "Stop slave SQL thread "
1256 "and wait to start backup until Slave_open_temp_tables in "
1257 "\"SHOW STATUS\" is zero. If there are no open temporary tables, "
1258 "the backup will take place, otherwise the SQL thread will be "
1259 "started and stopped until there are no open temporary tables. "
1260 "The backup will fail if Slave_open_temp_tables does not become "
1261 "zero after --safe-slave-backup-timeout seconds. The slave SQL "
1262 "thread will be restarted when the backup finishes.",
1263 (uchar *) &opt_safe_slave_backup, (uchar *) &opt_safe_slave_backup, 0,
1264 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1265
1266 {"rsync", OPT_RSYNC,
1267 "Uses the rsync utility to optimize local file "
1268 "transfers. When this option is specified, innobackupex uses rsync "
1269 "to copy all non-InnoDB files instead of spawning a separate cp for "
1270 "each file, which can be much faster for servers with a large number "
1271 "of databases or tables. This option cannot be used together with "
1272 "--stream.",
1273 (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0,
1274 0, 0, 0},
1275
1276 {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS,
1277 "This "
1278 "option, when specified, makes --copy-back or --move-back transfer "
1279 "files to non-empty directories. Note that no existing files will be "
1280 "overwritten. If --copy-back or --move-back has to copy a file from "
1281 "the backup directory which already exists in the destination "
1282 "directory, it will still fail with an error.",
1283 (uchar *) &opt_force_non_empty_dirs, (uchar *) &opt_force_non_empty_dirs,
1284 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1285
1286 {"no-version-check", OPT_NO_VERSION_CHECK,
1287 "This option disables the "
1288 "version check which is enabled by the --version-check option.",
1289 (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL,
1290 NO_ARG, 0, 0, 0, 0, 0, 0},
1291
1292 {"no-backup-locks", OPT_NO_BACKUP_LOCKS,
1293 "This option controls if "
1294 "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
1295 "on the backup stage. The option has no effect when backup locks are "
1296 "not supported by the server. This option is enabled by default, "
1297 "disable with --no-backup-locks.",
1298 (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0,
1299 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1300
1301 {"decompress", OPT_DECOMPRESS,
1302 "Decompresses all files with the .qp "
1303 "extension in a backup previously made with the --compress option.",
1304 (uchar *) &opt_decompress, (uchar *) &opt_decompress, 0, GET_BOOL, NO_ARG,
1305 0, 0, 0, 0, 0, 0},
1306
1307 {"user", 'u',
1308 "This option specifies the MySQL username used "
1309 "when connecting to the server, if that's not the current user. "
1310 "The option accepts a string argument. See mysql --help for details.",
1311 (uchar *) &opt_user, (uchar *) &opt_user, 0, GET_STR, REQUIRED_ARG, 0, 0,
1312 0, 0, 0, 0},
1313
1314 {"host", 'H',
1315 "This option specifies the host to use when "
1316 "connecting to the database server with TCP/IP. The option accepts "
1317 "a string argument. See mysql --help for details.",
1318 (uchar *) &opt_host, (uchar *) &opt_host, 0, GET_STR, REQUIRED_ARG, 0, 0,
1319 0, 0, 0, 0},
1320
1321 {"port", 'P',
1322 "This option specifies the port to use when "
1323 "connecting to the database server with TCP/IP. The option accepts "
1324 "a string argument. See mysql --help for details.",
1325 &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1326
1327 {"password", 'p',
1328 "This option specifies the password to use "
1329 "when connecting to the database. It accepts a string argument. "
1330 "See mysql --help for details.",
1331 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1332
1333 {"protocol", OPT_PROTOCOL,
1334 "The protocol to use for connection (tcp, socket, pipe, memory).", 0, 0,
1335 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1336
1337 {"socket", 'S',
1338 "This option specifies the socket to use when "
1339 "connecting to the local database server with a UNIX domain socket. "
1340 "The option accepts a string argument. See mysql --help for details.",
1341 (uchar *) &opt_socket, (uchar *) &opt_socket, 0, GET_STR, REQUIRED_ARG, 0,
1342 0, 0, 0, 0, 0},
1343
1344 {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME,
1345 "This option specifies the name of the backup series stored in the "
1346 "PERCONA_SCHEMA.xtrabackup_history history record to base an "
1347 "incremental backup on. Xtrabackup will search the history table "
1348 "looking for the most recent (highest innodb_to_lsn), successful "
1349 "backup in the series and take the to_lsn value to use as the "
1350 "starting lsn for the incremental backup. This will be mutually "
1351 "exclusive with --incremental-history-uuid, --incremental-basedir "
1352 "and --incremental-lsn. If no valid lsn can be found (no series by "
1353 "that name, no successful backups by that name), an error will be returned."
1354 " It is used with the --incremental option.",
1355 (uchar *) &opt_incremental_history_name,
1356 (uchar *) &opt_incremental_history_name, 0, GET_STR, REQUIRED_ARG, 0, 0,
1357 0, 0, 0, 0},
1358
1359 {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID,
1360 "This option specifies the UUID of the specific history record "
1361 "stored in the PERCONA_SCHEMA.xtrabackup_history to base an "
1362 "incremental backup on. --incremental-history-name, "
1363 "--incremental-basedir and --incremental-lsn. If no valid lsn can be "
1364 "found (no success record with that uuid), an error will be returned."
1365 " It is used with the --incremental option.",
1366 (uchar *) &opt_incremental_history_uuid,
1367 (uchar *) &opt_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0,
1368 0, 0, 0, 0},
1369
1370 {"remove-original", OPT_REMOVE_ORIGINAL,
1371 "Remove .qp files after decompression.", (uchar *) &opt_remove_original,
1372 (uchar *) &opt_remove_original, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1373
1374 {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
1375 "This option specifies which types of queries are allowed to complete "
1376 "before innobackupex will issue the global lock. Default is all.",
1377 (uchar *) &opt_lock_wait_query_type, (uchar *) &opt_lock_wait_query_type,
1378 &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0,
1379 0},
1380
1381 {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
1382 "This option specifies which types of queries should be killed to "
1383 "unblock the global lock. Default is \"all\".",
1384 (uchar *) &opt_kill_long_query_type, (uchar *) &opt_kill_long_query_type,
1385 &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0,
1386 0, 0},
1387
1388 {"history", OPT_HISTORY,
1389 "This option enables the tracking of backup history in the "
1390 "PERCONA_SCHEMA.xtrabackup_history table. An optional history "
1391 "series name may be specified that will be placed with the history "
1392 "record for the current backup being taken.",
1393 NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1394
1395 {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
1396 "This option specifies the number of seconds innobackupex waits "
1397 "between starting FLUSH TABLES WITH READ LOCK and killing those "
1398 "queries that block it. Default is 0 seconds, which means "
1399 "innobackupex will not attempt to kill any queries.",
1400 (uchar *) &opt_kill_long_queries_timeout,
1401 (uchar *) &opt_kill_long_queries_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0,
1402 0, 0, 0, 0},
1403
1404 {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
1405 "This option specifies time in seconds that innobackupex should wait "
1406 "for queries that would block FTWRL before running it. If there are "
1407 "still such queries when the timeout expires, innobackupex terminates "
1408 "with an error. Default is 0, in which case innobackupex does not "
1409 "wait for queries to complete and starts FTWRL immediately.",
1410 (uchar *) &opt_lock_wait_timeout, (uchar *) &opt_lock_wait_timeout, 0,
1411 GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1412
1413 {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
1414 "This option specifies the query run time threshold which is used by "
1415 "innobackupex to detect long-running queries with a non-zero value "
1416 "of --ftwrl-wait-timeout. FTWRL is not started until such "
1417 "long-running queries exist. This option has no effect if "
1418 "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
1419 (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0,
1420 GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0},
1421
1422
1423 {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
1424 "How many seconds --safe-slave-backup should wait for "
1425 "Slave_open_temp_tables to become zero. (default 300)",
1426 (uchar *) &opt_safe_slave_backup_timeout,
1427 (uchar *) &opt_safe_slave_backup_timeout, 0, GET_UINT, REQUIRED_ARG, 300,
1428 0, 0, 0, 0, 0},
1429
1430 {"binlog-info", OPT_BINLOG_INFO,
1431 "This option controls how backup should retrieve server's binary log "
1432 "coordinates corresponding to the backup. Possible values are OFF, ON, "
1433 "LOCKLESS and AUTO.",
1434 &opt_binlog_info, &opt_binlog_info, &binlog_info_typelib, GET_ENUM,
1435 OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0},
1436
1437 {"secure-auth", OPT_XB_SECURE_AUTH,
1438 "Refuse client connecting to server if it"
1439 " uses old (pre-4.1.1) protocol.",
1440 &opt_secure_auth, &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
1441 0},
1442
1443 {"log-innodb-page-corruption", OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
1444 "Continue backup if innodb corrupted pages are found. The pages are "
1445 "logged in " MB_CORRUPTED_PAGES_FILE
1446 " and backup is finished with error. "
1447 "--prepare will try to fix corrupted pages. If " MB_CORRUPTED_PAGES_FILE
1448 " exists after --prepare in base backup directory, backup still contains "
1449 "corrupted pages and can not be considered as consistent.",
1450 &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0,
1451 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1452
1453 #define MYSQL_CLIENT
1454 #include "sslopt-longopts.h"
1455 #undef MYSQL_CLIENT
1456
1457 {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
1458
1459 uint xb_client_options_count = array_elements(xb_client_options);
1460
1461 #ifndef DBUG_OFF
1462 /** Parameters to DBUG */
1463 static const char *dbug_option;
1464 #endif
1465
1466 struct my_option xb_server_options[] =
1467 {
1468 {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home,
1469 (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1470 {"tmpdir", 't',
1471 "Path for temporary files. Several paths may be specified, separated by a "
1472 #if defined(__WIN__) || defined(OS2) || defined(__NETWARE__)
1473 "semicolon (;)"
1474 #else
1475 "colon (:)"
1476 #endif
1477 ", in this case they are used in a round-robin fashion.",
1478 (G_PTR*) &opt_mysql_tmpdir,
1479 (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1480 {"parallel", OPT_XTRA_PARALLEL,
1481 "Number of threads to use for parallel datafiles transfer. "
1482 "The default value is 1.",
1483 (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
1484 REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
1485
1486 {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION,
1487 "Enable extended validation for Innodb data pages during backup phase. "
1488 "Will slow down backup considerably, in case encryption is used. "
1489 "May fail if tables are created during the backup.",
1490 (G_PTR*)&opt_extended_validation,
1491 (G_PTR*)&opt_extended_validation,
1492 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
1493
1494 {"encrypted_backup", OPT_XTRA_ENCRYPTED_BACKUP,
1495 "In --backup, assume that nonzero key_version implies that the page"
1496 " is encrypted. Use --backup --skip-encrypted-backup to allow"
1497 " copying unencrypted that were originally created before MySQL 5.1.48.",
1498 (G_PTR*)&opt_encrypted_backup,
1499 (G_PTR*)&opt_encrypted_backup,
1500 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0},
1501
1502 {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
1503 (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
1504 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1505
1506 {"log_bin", OPT_LOG, "Base name for the log sequence",
1507 &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1508
1509 {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility",
1510 (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0,
1511 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1512 #ifdef BTR_CUR_HASH_ADAPT
1513 {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX,
1514 "Enable InnoDB adaptive hash index (enabled by default). "
1515 "Disable with --skip-innodb-adaptive-hash-index.",
1516 &btr_search_enabled,
1517 &btr_search_enabled,
1518 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
1519 #endif /* BTR_CUR_HASH_ADAPT */
1520 {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT,
1521 "Data file autoextend increment in megabytes",
1522 (G_PTR*) &sys_tablespace_auto_extend_increment,
1523 (G_PTR*) &sys_tablespace_auto_extend_increment,
1524 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0},
1525 {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH,
1526 "Path to individual files and their sizes.", &innobase_data_file_path,
1527 &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1528 {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR,
1529 "The common part for InnoDB table spaces.", &innobase_data_home_dir,
1530 &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1531 {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE,
1532 "Enable InnoDB doublewrite buffer during --prepare.",
1533 (G_PTR*) &innobase_use_doublewrite,
1534 (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1535 {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY,
1536 "Number of IOPs the server can do. Tunes the background IO rate",
1537 (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity,
1538 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0},
1539 {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS,
1540 "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads,
1541 (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0,
1542 1, 0},
1543 {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS,
1544 "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads,
1545 (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
1546 1, 0},
1547 {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS,
1548 "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads,
1549 (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
1550 1, 0},
1551 {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE,
1552 "Stores each InnoDB table to an .ibd file in the database dir.",
1553 (G_PTR*) &innobase_file_per_table,
1554 (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG,
1555 FALSE, 0, 0, 0, 0, 0},
1556
1557 {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD,
1558 "With which method to flush data.",
1559 &srv_file_flush_method, &srv_file_flush_method,
1560 &innodb_flush_method_typelib, GET_ENUM, REQUIRED_ARG,
1561 IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_FSYNC), 0, 0, 0, 0, 0},
1562
1563 {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE,
1564 "The size of the buffer which InnoDB uses to write log to the log files on disk.",
1565 (G_PTR*) &srv_log_buffer_size, (G_PTR*) &srv_log_buffer_size, 0,
1566 GET_ULONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0},
1567 {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
1568 "Ignored for mysqld option compatibility",
1569 (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
1570 GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, log_group_max_size, 0,
1571 UNIV_PAGE_SIZE_MAX, 0},
1572 {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP,
1573 "Ignored for mysqld option compatibility",
1574 &srv_n_log_files, &srv_n_log_files,
1575 0, GET_LONG, REQUIRED_ARG, 1, 1, 100, 0, 1, 0},
1576 {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
1577 "Path to InnoDB log files.", &srv_log_group_home_dir,
1578 &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1579 {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT,
1580 "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct,
1581 (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0},
1582 {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO,
1583 "Use native AIO if supported on this platform.",
1584 (G_PTR*) &srv_use_native_aio,
1585 (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG,
1586 TRUE, 0, 0, 0, 0, 0},
1587 {"innodb_page_size", OPT_INNODB_PAGE_SIZE,
1588 "The universal page size of the database.",
1589 (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0,
1590 /* Use GET_LL to support numeric suffixes in 5.6 */
1591 GET_LL, REQUIRED_ARG,
1592 (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0},
1593 {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME,
1594 "Ignored for mysqld option compatibility",
1595 (G_PTR*) &innobase_buffer_pool_filename,
1596 (G_PTR*) &innobase_buffer_pool_filename,
1597 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1598
1599 #ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */
1600 {"dbug", '#', "Built in DBUG debugger.",
1601 &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG,
1602 0, 0, 0, 0, 0, 0},
1603 #endif
1604
1605 {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM,
1606 "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
1607 "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
1608 &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
1609 REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},
1610
1611 {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
1612 "Directory where undo tablespace files live, this path can be absolute.",
1613 &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0,
1614 0},
1615
1616 {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES,
1617 "Number of undo tablespaces to use.",
1618 (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces,
1619 0, GET_ULONG, REQUIRED_ARG, 0, 0, 126, 0, 1, 0},
1620
1621 {"innodb_compression_level", OPT_INNODB_COMPRESSION_LEVEL,
1622 "Compression level used for zlib compression.",
1623 (G_PTR*)&page_zip_level, (G_PTR*)&page_zip_level,
1624 0, GET_UINT, REQUIRED_ARG, 6, 0, 9, 0, 0, 0},
1625
1626 {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").",
1627 (G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
1628 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1629
1630 {"plugin-dir", OPT_PLUGIN_DIR,
1631 "Server plugin directory. Used to load encryption plugin during 'prepare' phase."
1632 "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)",
1633 &xb_plugin_dir, &xb_plugin_dir,
1634 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1635
1636 {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS,
1637 "Whether to require checksums for InnoDB redo log blocks",
1638 &innodb_log_checksums, &innodb_log_checksums,
1639 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 },
1640
1641 {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
1642 "descriptors to reserve with setrlimit().",
1643 (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
1644 REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},
1645
1646 {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table "
1647 "before backup starts to copy it and until the backup is completed.",
1648 (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
1649 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1650
1651 {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
1652 "This option is only used with --copy-back or --move-back option",
1653 &xb_rocksdb_datadir, &xb_rocksdb_datadir,
1654 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1655
1656 { "rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
1657 "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
1658 &xb_backup_rocksdb, &xb_backup_rocksdb,
1659 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
1660
1661 {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user "
1662 "privileges fro the backup user",
1663 &opt_check_privileges, &opt_check_privileges,
1664 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
1665
1666 {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY,
1667 "(for --prepare): Crash recovery mode (ignores "
1668 "page corruption; for emergencies only).",
1669 (G_PTR*)&srv_force_recovery,
1670 (G_PTR*)&srv_force_recovery,
1671 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
1672
1673 { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
1674 };
1675
1676 uint xb_server_options_count = array_elements(xb_server_options);
1677
1678
1679 static std::set<std::string> tables_for_export;
1680
append_export_table(const char * dbname,const char * tablename,bool is_remote,bool set_size)1681 static void append_export_table(const char *dbname, const char *tablename,
1682 bool is_remote, bool set_size)
1683 {
1684 if(dbname && tablename && !is_remote)
1685 {
1686 char buf[3*FN_REFLEN];
1687 snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename);
1688 // trim .ibd
1689 char *p=strrchr(buf, '.');
1690 if (p) *p=0;
1691
1692 std::string name=ut_get_name(0, buf);
1693 /* Strip partition name comment from table name, if any */
1694 if (ends_with(name.c_str(), "*/"))
1695 {
1696 size_t pos= name.rfind("/*");
1697 if (pos != std::string::npos)
1698 name.resize(pos);
1699 }
1700 tables_for_export.insert(name);
1701 }
1702 }
1703
1704
1705 #define BOOTSTRAP_FILENAME "mariabackup_prepare_for_export.sql"
1706
create_bootstrap_file()1707 static int create_bootstrap_file()
1708 {
1709 FILE *f= fopen(BOOTSTRAP_FILENAME,"wb");
1710 if(!f)
1711 return -1;
1712
1713 fputs("SET NAMES UTF8;\n",f);
1714 enumerate_ibd_files(append_export_table);
1715 for (std::set<std::string>::iterator it = tables_for_export.begin();
1716 it != tables_for_export.end(); it++)
1717 {
1718 const char *tab = it->c_str();
1719 fprintf(f,
1720 "BEGIN NOT ATOMIC "
1721 "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;"
1722 "FLUSH TABLES %s FOR EXPORT;"
1723 "END;\n"
1724 "UNLOCK TABLES;\n",
1725 tab);
1726 }
1727 fclose(f);
1728 return 0;
1729 }
1730
prepare_export()1731 static int prepare_export()
1732 {
1733 int err= -1;
1734
1735 char cmdline[2*FN_REFLEN];
1736 FILE *outf;
1737
1738 if (create_bootstrap_file())
1739 return -1;
1740
1741 // Process defaults-file , it can have some --lc-language stuff,
1742 // which is* unfortunately* still necessary to get mysqld up
1743 if (strncmp(orig_argv1,"--defaults-file=", 16) == 0)
1744 {
1745 snprintf(cmdline, sizeof cmdline,
1746 IF_WIN("\"","") "\"%s\" --mysqld \"%s\""
1747 " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1748 " --innodb --innodb-fast-shutdown=0 --loose-partition"
1749 " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1750 " --console --skip-log-error --skip-log-bin --bootstrap %s< "
1751 BOOTSTRAP_FILENAME IF_WIN("\"",""),
1752 mariabackup_exe,
1753 orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
1754 xtrabackup_use_memory,
1755 (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
1756 }
1757 else
1758 {
1759 snprintf(cmdline, sizeof cmdline,
1760 IF_WIN("\"","") "\"%s\" --mysqld"
1761 " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1762 " --innodb --innodb-fast-shutdown=0 --loose-partition"
1763 " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1764 " --console --log-error= --skip-log-bin --bootstrap %s< "
1765 BOOTSTRAP_FILENAME IF_WIN("\"",""),
1766 mariabackup_exe,
1767 (my_defaults_group_suffix?my_defaults_group_suffix:""),
1768 xtrabackup_use_memory,
1769 (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
1770 }
1771
1772 msg("Prepare export : executing %s\n", cmdline);
1773 fflush(stderr);
1774
1775 outf= popen(cmdline,"r");
1776 if (!outf)
1777 goto end;
1778
1779 char outline[FN_REFLEN];
1780 while(fgets(outline, sizeof(outline)-1, outf))
1781 fprintf(stderr,"%s",outline);
1782
1783 err = pclose(outf);
1784 end:
1785 unlink(BOOTSTRAP_FILENAME);
1786 return err;
1787 }
1788
1789
1790 static const char *xb_client_default_groups[]={
1791 "xtrabackup", "mariabackup",
1792 "client", "client-server",
1793 "client-mariadb",
1794 0, 0, 0
1795 };
1796
1797 static const char *xb_server_default_groups[]={
1798 "xtrabackup", "mariabackup",
1799 "mysqld", "server", MYSQL_BASE_VERSION,
1800 "mariadb", MARIADB_BASE_VERSION,
1801 "client-server",
1802 #ifdef WITH_WSREP
1803 "galera",
1804 #endif
1805 0, 0, 0
1806 };
1807
print_version(void)1808 static void print_version(void)
1809 {
1810 fprintf(stderr, "%s based on MariaDB server %s %s (%s)\n",
1811 my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
1812 }
1813
usage(void)1814 static void usage(void)
1815 {
1816 puts("Open source backup tool for InnoDB and XtraDB\n\
1817 \n\
1818 Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\
1819 Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\
1820 \n\
1821 This program is free software; you can redistribute it and/or\n\
1822 modify it under the terms of the GNU General Public License\n\
1823 as published by the Free Software Foundation version 2\n\
1824 of the License.\n\
1825 \n\
1826 This program is distributed in the hope that it will be useful,\n\
1827 but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
1828 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
1829 GNU General Public License for more details.\n\
1830 \n\
1831 You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n");
1832
1833 printf("Usage: %s [--defaults-file=#] [--backup | --prepare | --copy-back | --move-back] [OPTIONS]\n",my_progname);
1834 print_defaults("my", xb_server_default_groups);
1835 my_print_help(xb_client_options);
1836 my_print_help(xb_server_options);
1837 my_print_variables(xb_server_options);
1838 my_print_variables(xb_client_options);
1839 }
1840
1841 #define ADD_PRINT_PARAM_OPT(value) \
1842 { \
1843 print_param_str << opt->name << "=" << value << "\n"; \
1844 param_set.insert(opt->name); \
1845 }
1846
1847 /************************************************************************
1848 Check if parameter is set in defaults file or via command line argument
1849 @return true if parameter is set. */
1850 bool
check_if_param_set(const char * param)1851 check_if_param_set(const char *param)
1852 {
1853 return param_set.find(param) != param_set.end();
1854 }
1855
1856 my_bool
xb_get_one_option(int optid,const struct my_option * opt,char * argument)1857 xb_get_one_option(int optid,
1858 const struct my_option *opt __attribute__((unused)),
1859 char *argument)
1860 {
1861 switch(optid) {
1862 case 'h':
1863 strmake(mysql_real_data_home,argument, FN_REFLEN - 1);
1864 mysql_data_home= mysql_real_data_home;
1865
1866 ADD_PRINT_PARAM_OPT(mysql_real_data_home);
1867 break;
1868
1869 case 't':
1870
1871 ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir);
1872 break;
1873
1874 case OPT_INNODB_DATA_HOME_DIR:
1875
1876 ADD_PRINT_PARAM_OPT(innobase_data_home_dir);
1877 break;
1878
1879 case OPT_INNODB_DATA_FILE_PATH:
1880
1881 ADD_PRINT_PARAM_OPT(innobase_data_file_path);
1882 break;
1883
1884 case OPT_INNODB_LOG_GROUP_HOME_DIR:
1885
1886 ADD_PRINT_PARAM_OPT(srv_log_group_home_dir);
1887 break;
1888
1889 case OPT_INNODB_LOG_FILES_IN_GROUP:
1890 case OPT_INNODB_LOG_FILE_SIZE:
1891 break;
1892
1893 case OPT_INNODB_FLUSH_METHOD:
1894 ut_a(srv_file_flush_method
1895 <= IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT_NO_FSYNC));
1896 ADD_PRINT_PARAM_OPT(innodb_flush_method_names[srv_file_flush_method]);
1897 break;
1898
1899 case OPT_INNODB_PAGE_SIZE:
1900
1901 ADD_PRINT_PARAM_OPT(innobase_page_size);
1902 break;
1903
1904 case OPT_INNODB_UNDO_DIRECTORY:
1905
1906 ADD_PRINT_PARAM_OPT(srv_undo_dir);
1907 break;
1908
1909 case OPT_INNODB_UNDO_TABLESPACES:
1910
1911 ADD_PRINT_PARAM_OPT(srv_undo_tablespaces);
1912 break;
1913
1914 case OPT_INNODB_CHECKSUM_ALGORITHM:
1915
1916 ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE);
1917
1918 ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]);
1919 break;
1920
1921 case OPT_INNODB_COMPRESSION_LEVEL:
1922 ADD_PRINT_PARAM_OPT(page_zip_level);
1923 break;
1924
1925 case OPT_INNODB_BUFFER_POOL_FILENAME:
1926
1927 ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
1928 break;
1929
1930 case OPT_INNODB_FORCE_RECOVERY:
1931
1932 if (srv_force_recovery) {
1933 ADD_PRINT_PARAM_OPT(srv_force_recovery);
1934 }
1935 break;
1936
1937 case OPT_XTRA_TARGET_DIR:
1938 strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
1939 xtrabackup_target_dir= xtrabackup_real_target_dir;
1940 break;
1941 case OPT_XTRA_STREAM:
1942 if (!strcasecmp(argument, "mbstream") ||
1943 !strcasecmp(argument, "xbstream"))
1944 xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
1945 else
1946 {
1947 msg("Invalid --stream argument: %s", argument);
1948 return 1;
1949 }
1950 xtrabackup_stream = TRUE;
1951 break;
1952 case OPT_XTRA_COMPRESS:
1953 if (argument == NULL)
1954 xtrabackup_compress_alg = "quicklz";
1955 else if (strcasecmp(argument, "quicklz"))
1956 {
1957 msg("Invalid --compress argument: %s", argument);
1958 return 1;
1959 }
1960 xtrabackup_compress = TRUE;
1961 break;
1962 case OPT_DECOMPRESS:
1963 opt_decompress = TRUE;
1964 xtrabackup_decrypt_decompress = true;
1965 break;
1966 case (int) OPT_CORE_FILE:
1967 test_flags |= TEST_CORE_ON_SIGNAL;
1968 break;
1969 case OPT_HISTORY:
1970 if (argument) {
1971 opt_history = argument;
1972 } else {
1973 opt_history = "";
1974 }
1975 break;
1976 case 'p':
1977 if (argument)
1978 {
1979 char *start= argument;
1980 my_free(opt_password);
1981 opt_password= my_strdup(argument, MYF(MY_FAE));
1982 while (*argument) *argument++= 'x'; // Destroy argument
1983 if (*start)
1984 start[1]=0 ;
1985 }
1986 break;
1987 case OPT_PROTOCOL:
1988 if (argument)
1989 {
1990 if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib,
1991 opt->name)) <= 0)
1992 {
1993 sf_leaking_memory= 1; /* no memory leak reports here */
1994 exit(1);
1995 }
1996 }
1997 break;
1998 #define MYSQL_CLIENT
1999 #include "sslopt-case.h"
2000 #undef MYSQL_CLIENT
2001
2002 case '?':
2003 usage();
2004 exit(EXIT_SUCCESS);
2005 break;
2006 case 'v':
2007 print_version();
2008 exit(EXIT_SUCCESS);
2009 break;
2010 default:
2011 break;
2012 }
2013 return 0;
2014 }
2015
innodb_init_param()2016 static bool innodb_init_param()
2017 {
2018 srv_is_being_started = TRUE;
2019 /* === some variables from mysqld === */
2020 memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));
2021
2022 if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) {
2023 msg("init_tmpdir() failed");
2024 return true;
2025 }
2026 xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list);
2027 /* dummy for initialize all_charsets[] */
2028 get_charset_name(0);
2029
2030 srv_page_size = 0;
2031 srv_page_size_shift = 0;
2032 #ifdef BTR_CUR_HASH_ADAPT
2033 btr_ahi_parts = 1;
2034 #endif /* BTR_CUR_HASH_ADAPT */
2035
2036 if (innobase_page_size != (1LL << 14)) {
2037 size_t n_shift = get_bit_shift(size_t(innobase_page_size));
2038
2039 if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
2040 srv_page_size_shift = ulong(n_shift);
2041 srv_page_size = 1U << n_shift;
2042 msg("InnoDB: The universal page size of the "
2043 "database is set to %lu.", srv_page_size);
2044 } else {
2045 msg("invalid value of "
2046 "innobase_page_size: %lld", innobase_page_size);
2047 goto error;
2048 }
2049 } else {
2050 srv_page_size_shift = 14;
2051 srv_page_size = 1U << 14;
2052 }
2053
2054 /* Check that values don't overflow on 32-bit systems. */
2055 if (sizeof(ulint) == 4) {
2056 if (xtrabackup_use_memory > UINT_MAX32) {
2057 msg("mariabackup: use-memory can't be over 4GB"
2058 " on 32-bit systems");
2059 }
2060 }
2061
2062 static char default_path[2] = { FN_CURLIB, 0 };
2063 fil_path_to_mysql_datadir = default_path;
2064
2065 /* Set InnoDB initialization parameters according to the values
2066 read from MySQL .cnf file */
2067
2068 if (xtrabackup_backup) {
2069 msg("mariabackup: using the following InnoDB configuration:");
2070 } else {
2071 msg("mariabackup: using the following InnoDB configuration "
2072 "for recovery:");
2073 }
2074
2075 /*--------------- Data files -------------------------*/
2076
2077 /* The default dir for data files is the datadir of MySQL */
2078
2079 srv_data_home = (xtrabackup_backup && innobase_data_home_dir
2080 ? innobase_data_home_dir : default_path);
2081 msg("innodb_data_home_dir = %s", srv_data_home);
2082
2083 /* Set default InnoDB data file size to 10 MB and let it be
2084 auto-extending. Thus users can use InnoDB in >= 4.0 without having
2085 to specify any startup options. */
2086
2087 if (!innobase_data_file_path) {
2088 innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
2089 }
2090 msg("innodb_data_file_path = %s",
2091 innobase_data_file_path);
2092
2093 /* This is the first time univ_page_size is used.
2094 It was initialized to 16k pages before srv_page_size was set */
2095 univ_page_size.copy_from(
2096 page_size_t(srv_page_size, srv_page_size, false));
2097
2098 srv_sys_space.set_space_id(TRX_SYS_SPACE);
2099 srv_sys_space.set_name("innodb_system");
2100 srv_sys_space.set_path(srv_data_home);
2101 srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE());
2102
2103 if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
2104 goto error;
2105 }
2106
2107 srv_sys_space.normalize_size();
2108 srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);
2109
2110 /* -------------- Log files ---------------------------*/
2111
2112 /* The default dir for log files is the datadir of MySQL */
2113
2114 if (!(xtrabackup_backup && srv_log_group_home_dir)) {
2115 srv_log_group_home_dir = default_path;
2116 }
2117 if (xtrabackup_prepare && xtrabackup_incremental_dir) {
2118 srv_log_group_home_dir = xtrabackup_incremental_dir;
2119 }
2120 msg("innodb_log_group_home_dir = %s",
2121 srv_log_group_home_dir);
2122
2123 os_normalize_path(srv_log_group_home_dir);
2124
2125 if (strchr(srv_log_group_home_dir, ';')) {
2126 msg("syntax error in innodb_log_group_home_dir, ");
2127 goto error;
2128 }
2129
2130 srv_adaptive_flushing = FALSE;
2131
2132 /* We set srv_pool_size here in units of 1 kB. InnoDB internally
2133 changes the value so that it becomes the number of database pages. */
2134
2135 srv_buf_pool_size = (ulint) xtrabackup_use_memory;
2136 srv_buf_pool_chunk_unit = (ulong)srv_buf_pool_size;
2137 srv_buf_pool_instances = 1;
2138 srv_n_page_cleaners = 1;
2139
2140 srv_n_file_io_threads = (ulint) innobase_file_io_threads;
2141 srv_n_read_io_threads = (ulint) innobase_read_io_threads;
2142 srv_n_write_io_threads = (ulint) innobase_write_io_threads;
2143
2144 srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;
2145
2146 row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout;
2147
2148 srv_file_per_table = (my_bool) innobase_file_per_table;
2149
2150 srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog;
2151
2152 srv_max_n_open_files = ULINT_UNDEFINED - 5;
2153 srv_innodb_status = (ibool) innobase_create_status_file;
2154
2155 srv_print_verbose_log = verbose ? 2 : 1;
2156
2157 /* Store the default charset-collation number of this MySQL
2158 installation */
2159
2160 /* We cannot treat characterset here for now!! */
2161 data_mysql_default_charset_coll = (ulint)default_charset_info->number;
2162
2163 ut_ad(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number);
2164
2165 #ifdef _WIN32
2166 srv_use_native_aio = TRUE;
2167
2168 #elif defined(LINUX_NATIVE_AIO)
2169
2170 if (srv_use_native_aio) {
2171 msg("InnoDB: Using Linux native AIO");
2172 }
2173 #else
2174 /* Currently native AIO is supported only on windows and linux
2175 and that also when the support is compiled in. In all other
2176 cases, we ignore the setting of innodb_use_native_aio. */
2177 srv_use_native_aio = FALSE;
2178
2179 #endif
2180
2181 /* Assign the default value to srv_undo_dir if it's not specified, as
2182 my_getopt does not support default values for string options. We also
2183 ignore the option and override innodb_undo_directory on --prepare,
2184 because separate undo tablespaces are copied to the root backup
2185 directory. */
2186
2187 if (!srv_undo_dir || !xtrabackup_backup) {
2188 srv_undo_dir = (char*) ".";
2189 }
2190
2191 compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1);
2192
2193 /*
2194 * This option can be read both from the command line, and the
2195 * defaults file. The assignment should account for both cases,
2196 * and for "--innobackupex". Since the command line argument is
2197 * parsed after the defaults file, it takes precedence.
2198 */
2199 if (xtrabackup_innodb_force_recovery) {
2200 srv_force_recovery = xtrabackup_innodb_force_recovery;
2201 }
2202
2203 if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) {
2204 if (!xtrabackup_prepare) {
2205 msg("mariabackup: The option \"innodb_force_recovery\""
2206 " should only be used with \"%s\".",
2207 (innobackupex_mode ? "--apply-log" : "--prepare"));
2208 goto error;
2209 } else {
2210 msg("innodb_force_recovery = %lu", srv_force_recovery);
2211 }
2212 }
2213
2214 #ifdef _WIN32
2215 srv_use_native_aio = TRUE;
2216 #endif
2217 return false;
2218
2219 error:
2220 msg("mariabackup: innodb_init_param(): Error occurred.\n");
2221 return true;
2222 }
2223
innodb_init()2224 static bool innodb_init()
2225 {
2226 bool create_new_db = false;
2227 /* Check if the data files exist or not. */
2228 dberr_t err = srv_sys_space.check_file_spec(&create_new_db, 5U << 20);
2229
2230 if (err == DB_SUCCESS) {
2231 err = srv_start(create_new_db);
2232 }
2233
2234 if (err != DB_SUCCESS) {
2235 die("mariabackup: innodb_init() returned %d (%s).",
2236 err, ut_strerr(err));
2237 }
2238
2239 return(FALSE);
2240 }
2241
2242 /* ================= common ================= */
2243
2244 /***********************************************************************
2245 Read backup meta info.
2246 @return TRUE on success, FALSE on failure. */
2247 static
2248 my_bool
xtrabackup_read_metadata(char * filename)2249 xtrabackup_read_metadata(char *filename)
2250 {
2251 FILE *fp;
2252 my_bool r = TRUE;
2253
2254 fp = fopen(filename,"r");
2255 if(!fp) {
2256 msg("Error: cannot open %s", filename);
2257 return(FALSE);
2258 }
2259
2260 if (fscanf(fp, "backup_type = %29s\n", metadata_type)
2261 != 1) {
2262 r = FALSE;
2263 goto end;
2264 }
2265 /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
2266 format. */
2267 if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn)
2268 != 1) {
2269 r = FALSE;
2270 goto end;
2271 }
2272 if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn)
2273 != 1) {
2274 r = FALSE;
2275 goto end;
2276 }
2277 if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn)
2278 != 1) {
2279 metadata_last_lsn = 0;
2280 }
2281 /* Optional fields */
2282
2283 end:
2284 fclose(fp);
2285
2286 return(r);
2287 }
2288
2289 /***********************************************************************
2290 Print backup meta info to a specified buffer. */
2291 static
2292 void
xtrabackup_print_metadata(char * buf,size_t buf_len)2293 xtrabackup_print_metadata(char *buf, size_t buf_len)
2294 {
2295 /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
2296 format. */
2297 snprintf(buf, buf_len,
2298 "backup_type = %s\n"
2299 "from_lsn = " UINT64PF "\n"
2300 "to_lsn = " UINT64PF "\n"
2301 "last_lsn = " UINT64PF "\n",
2302 metadata_type,
2303 metadata_from_lsn,
2304 metadata_to_lsn,
2305 metadata_last_lsn);
2306 }
2307
2308 /***********************************************************************
2309 Stream backup meta info to a specified datasink.
2310 @return TRUE on success, FALSE on failure. */
2311 static
2312 my_bool
xtrabackup_stream_metadata(ds_ctxt_t * ds_ctxt)2313 xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt)
2314 {
2315 char buf[1024];
2316 size_t len;
2317 ds_file_t *stream;
2318 MY_STAT mystat;
2319 my_bool rc = TRUE;
2320
2321 xtrabackup_print_metadata(buf, sizeof(buf));
2322
2323 len = strlen(buf);
2324
2325 mystat.st_size = len;
2326 mystat.st_mtime = my_time(0);
2327
2328 stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat);
2329 if (stream == NULL) {
2330 msg("Error: cannot open output stream for %s", XTRABACKUP_METADATA_FILENAME);
2331 return(FALSE);
2332 }
2333
2334 if (ds_write(stream, buf, len)) {
2335 rc = FALSE;
2336 }
2337
2338 if (ds_close(stream)) {
2339 rc = FALSE;
2340 }
2341
2342 return(rc);
2343 }
2344
2345 /***********************************************************************
2346 Write backup meta info to a specified file.
2347 @return TRUE on success, FALSE on failure. */
2348 static
2349 my_bool
xtrabackup_write_metadata(const char * filepath)2350 xtrabackup_write_metadata(const char *filepath)
2351 {
2352 char buf[1024];
2353 size_t len;
2354 FILE *fp;
2355
2356 xtrabackup_print_metadata(buf, sizeof(buf));
2357
2358 len = strlen(buf);
2359
2360 fp = fopen(filepath, "w");
2361 if(!fp) {
2362 msg("Error: cannot open %s", filepath);
2363 return(FALSE);
2364 }
2365 if (fwrite(buf, len, 1, fp) < 1) {
2366 fclose(fp);
2367 return(FALSE);
2368 }
2369
2370 fclose(fp);
2371
2372 return(TRUE);
2373 }
2374
2375 /***********************************************************************
2376 Read meta info for an incremental delta.
2377 @return TRUE on success, FALSE on failure. */
2378 static my_bool
xb_read_delta_metadata(const char * filepath,xb_delta_info_t * info)2379 xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info)
2380 {
2381 FILE* fp;
2382 char key[51];
2383 char value[51];
2384 my_bool r = TRUE;
2385
2386 /* set defaults */
2387 ulint page_size = ULINT_UNDEFINED, zip_size = 0;
2388 info->space_id = ULINT_UNDEFINED;
2389
2390 fp = fopen(filepath, "r");
2391 if (!fp) {
2392 /* Meta files for incremental deltas are optional */
2393 return(TRUE);
2394 }
2395
2396 while (!feof(fp)) {
2397 if (fscanf(fp, "%50s = %50s\n", key, value) == 2) {
2398 if (strcmp(key, "page_size") == 0) {
2399 page_size = strtoul(value, NULL, 10);
2400 } else if (strcmp(key, "zip_size") == 0) {
2401 zip_size = strtoul(value, NULL, 10);
2402 } else if (strcmp(key, "space_id") == 0) {
2403 info->space_id = strtoul(value, NULL, 10);
2404 }
2405 }
2406 }
2407
2408 fclose(fp);
2409
2410 if (page_size == ULINT_UNDEFINED) {
2411 msg("page_size is required in %s", filepath);
2412 r = FALSE;
2413 } else {
2414 info->page_size = page_size_t(zip_size ? zip_size : page_size,
2415 page_size, zip_size != 0);
2416 }
2417
2418 if (info->space_id == ULINT_UNDEFINED) {
2419 msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 "
2420 "or earlier, some DDL operations between full and incremental "
2421 "backups may be handled incorrectly");
2422 }
2423
2424 return(r);
2425 }
2426
2427 /***********************************************************************
2428 Write meta info for an incremental delta.
2429 @return TRUE on success, FALSE on failure. */
2430 my_bool
xb_write_delta_metadata(const char * filename,const xb_delta_info_t * info)2431 xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info)
2432 {
2433 ds_file_t *f;
2434 char buf[64];
2435 my_bool ret;
2436 size_t len;
2437 MY_STAT mystat;
2438
2439 snprintf(buf, sizeof(buf),
2440 "page_size = " ULINTPF "\n"
2441 "zip_size = " ULINTPF " \n"
2442 "space_id = " ULINTPF "\n",
2443 info->page_size.logical(),
2444 info->page_size.is_compressed()
2445 ? info->page_size.physical() : 0,
2446 info->space_id);
2447 len = strlen(buf);
2448
2449 mystat.st_size = len;
2450 mystat.st_mtime = my_time(0);
2451
2452 f = ds_open(ds_meta, filename, &mystat);
2453 if (f == NULL) {
2454 msg("Error: Can't open output stream for %s",filename);
2455 return(FALSE);
2456 }
2457
2458 ret = (ds_write(f, buf, len) == 0);
2459
2460 if (ds_close(f)) {
2461 ret = FALSE;
2462 }
2463
2464 return(ret);
2465 }
2466
2467 /* ================= backup ================= */
2468 void
xtrabackup_io_throttling(void)2469 xtrabackup_io_throttling(void)
2470 {
2471 if (xtrabackup_backup && xtrabackup_throttle && (io_ticket--) < 0) {
2472 os_event_reset(wait_throttle);
2473 os_event_wait(wait_throttle);
2474 }
2475 }
2476
2477 static
regex_list_check_match(const regex_list_t & list,const char * name)2478 my_bool regex_list_check_match(
2479 const regex_list_t& list,
2480 const char* name)
2481 {
2482 regmatch_t tables_regmatch[1];
2483 for (regex_list_t::const_iterator i = list.begin(), end = list.end();
2484 i != end; ++i) {
2485 const regex_t& regex = *i;
2486 int regres = regexec(®ex, name, 1, tables_regmatch, 0);
2487
2488 if (regres != REG_NOMATCH) {
2489 return(TRUE);
2490 }
2491 }
2492 return(FALSE);
2493 }
2494
2495 static
2496 my_bool
find_filter_in_hashtable(const char * name,hash_table_t * table,xb_filter_entry_t ** result)2497 find_filter_in_hashtable(
2498 const char* name,
2499 hash_table_t* table,
2500 xb_filter_entry_t** result
2501 )
2502 {
2503 xb_filter_entry_t* found = NULL;
2504 HASH_SEARCH(name_hash, table, ut_fold_string(name),
2505 xb_filter_entry_t*,
2506 found, (void) 0,
2507 !strcmp(found->name, name));
2508
2509 if (found && result) {
2510 *result = found;
2511 }
2512 return (found != NULL);
2513 }
2514
2515 /************************************************************************
2516 Checks if a given table name matches any of specifications given in
2517 regex_list or tables_hash.
2518
2519 @return TRUE on match or both regex_list and tables_hash are empty.*/
2520 static my_bool
check_if_table_matches_filters(const char * name,const regex_list_t & regex_list,hash_table_t * tables_hash)2521 check_if_table_matches_filters(const char *name,
2522 const regex_list_t& regex_list,
2523 hash_table_t* tables_hash)
2524 {
2525 if (regex_list.empty() && !tables_hash) {
2526 return(FALSE);
2527 }
2528
2529 if (regex_list_check_match(regex_list, name)) {
2530 return(TRUE);
2531 }
2532
2533 if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
2534 return(TRUE);
2535 }
2536
2537 return FALSE;
2538 }
2539
2540 enum skip_database_check_result {
2541 DATABASE_SKIP,
2542 DATABASE_SKIP_SOME_TABLES,
2543 DATABASE_DONT_SKIP,
2544 DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
2545 };
2546
2547 /************************************************************************
2548 Checks if a database specified by name should be skipped from backup based on
2549 the --databases, --databases_file or --databases_exclude options.
2550
2551 @return TRUE if entire database should be skipped,
2552 FALSE otherwise.
2553 */
2554 static
2555 skip_database_check_result
check_if_skip_database(const char * name)2556 check_if_skip_database(
2557 const char* name /*!< in: path to the database */
2558 )
2559 {
2560 /* There are some filters for databases, check them */
2561 xb_filter_entry_t* database = NULL;
2562
2563 if (databases_exclude_hash &&
2564 find_filter_in_hashtable(name, databases_exclude_hash,
2565 &database) &&
2566 (!database->has_tables || !databases_include_hash)) {
2567 /* Database is found and there are no tables specified,
2568 skip entire db. */
2569 return DATABASE_SKIP;
2570 }
2571
2572 if (databases_include_hash) {
2573 if (!find_filter_in_hashtable(name, databases_include_hash,
2574 &database)) {
2575 /* Database isn't found, skip the database */
2576 return DATABASE_SKIP;
2577 } else if (database->has_tables) {
2578 return DATABASE_SKIP_SOME_TABLES;
2579 } else {
2580 return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
2581 }
2582 }
2583
2584 return DATABASE_DONT_SKIP;
2585 }
2586
2587 /************************************************************************
2588 Checks if a database specified by path should be skipped from backup based on
2589 the --databases, --databases_file or --databases_exclude options.
2590
2591 @return TRUE if the table should be skipped. */
2592 my_bool
check_if_skip_database_by_path(const char * path)2593 check_if_skip_database_by_path(
2594 const char* path /*!< in: path to the db directory. */
2595 )
2596 {
2597 if (databases_include_hash == NULL &&
2598 databases_exclude_hash == NULL) {
2599 return(FALSE);
2600 }
2601
2602 const char* db_name = strrchr(path, OS_PATH_SEPARATOR);
2603 if (db_name == NULL) {
2604 db_name = path;
2605 } else {
2606 ++db_name;
2607 }
2608
2609 return check_if_skip_database(db_name) == DATABASE_SKIP;
2610 }
2611
2612 /************************************************************************
2613 Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
2614 or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on
2615 the --tables or --tables-file options.
2616
2617 @return TRUE if the table should be skipped. */
2618 my_bool
check_if_skip_table(const char * name)2619 check_if_skip_table(
2620 /******************/
2621 const char* name) /*!< in: path to the table */
2622 {
2623 char buf[FN_REFLEN];
2624 const char *dbname, *tbname;
2625 const char *ptr;
2626 char *eptr;
2627
2628 if (regex_exclude_list.empty() &&
2629 regex_include_list.empty() &&
2630 tables_include_hash == NULL &&
2631 tables_exclude_hash == NULL &&
2632 databases_include_hash == NULL &&
2633 databases_exclude_hash == NULL) {
2634 return(FALSE);
2635 }
2636
2637 dbname = NULL;
2638 tbname = name;
2639 while ((ptr = strchr(tbname, '/')) != NULL) {
2640 dbname = tbname;
2641 tbname = ptr + 1;
2642 }
2643
2644 if (dbname == NULL) {
2645 return(FALSE);
2646 }
2647
2648 strncpy(buf, dbname, FN_REFLEN - 1);
2649 buf[FN_REFLEN - 1] = '\0';
2650 buf[tbname - 1 - dbname] = '\0';
2651
2652 const skip_database_check_result skip_database =
2653 check_if_skip_database(buf);
2654 if (skip_database == DATABASE_SKIP) {
2655 return (TRUE);
2656 }
2657
2658 buf[tbname - 1 - dbname] = '.';
2659
2660 /* Check if there's a suffix in the table name. If so, truncate it. We
2661 rely on the fact that a dot cannot be a part of a table name (it is
2662 encoded by the server with the @NNNN syntax). */
2663 if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) {
2664
2665 *eptr = '\0';
2666 }
2667
2668 /* For partitioned tables first try to match against the regexp
2669 without truncating the #P#... suffix so we can backup individual
2670 partitions with regexps like '^test[.]t#P#p5' */
2671 if (check_if_table_matches_filters(buf, regex_exclude_list,
2672 tables_exclude_hash)) {
2673 return(TRUE);
2674 }
2675 if (check_if_table_matches_filters(buf, regex_include_list,
2676 tables_include_hash)) {
2677 return(FALSE);
2678 }
2679 if ((eptr = strstr(buf, "#P#")) != NULL) {
2680 *eptr = 0;
2681
2682 if (check_if_table_matches_filters(buf, regex_exclude_list,
2683 tables_exclude_hash)) {
2684 return (TRUE);
2685 }
2686 if (check_if_table_matches_filters(buf, regex_include_list,
2687 tables_include_hash)) {
2688 return(FALSE);
2689 }
2690 }
2691
2692 if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
2693 /* Database is in include-list, and qualified name wasn't
2694 found in any of exclusion filters.*/
2695 return (FALSE);
2696 }
2697
2698 if (skip_database == DATABASE_SKIP_SOME_TABLES ||
2699 !regex_include_list.empty() ||
2700 tables_include_hash) {
2701
2702 /* Include lists are present, but qualified name
2703 failed to match any.*/
2704 return(TRUE);
2705 }
2706
2707 return(FALSE);
2708 }
2709
2710 const char*
xb_get_copy_action(const char * dflt)2711 xb_get_copy_action(const char *dflt)
2712 {
2713 const char *action;
2714
2715 if (xtrabackup_stream) {
2716 if (xtrabackup_compress) {
2717 action = "Compressing and streaming";
2718 } else {
2719 action = "Streaming";
2720 }
2721 } else {
2722 if (xtrabackup_compress) {
2723 action = "Compressing";
2724 } else {
2725 action = dflt;
2726 }
2727 }
2728
2729 return(action);
2730 }
2731
2732
2733 /** Copy innodb data file to the specified destination.
2734
2735 @param[in] node file node of a tablespace
2736 @param[in] thread_n thread id, used in the text of diagnostic messages
2737 @param[in] dest_name destination file name
2738 @param[in] write_filter write filter to copy data, can be pass-through filter
2739 for full backup, pages filter for incremental backup, etc.
2740
2741 @return FALSE on success and TRUE on error */
xtrabackup_copy_datafile(fil_node_t * node,uint thread_n,const char * dest_name,const xb_write_filt_t & write_filter,CorruptedPages & corrupted_pages)2742 static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n,
2743 const char *dest_name,
2744 const xb_write_filt_t &write_filter,
2745 CorruptedPages &corrupted_pages)
2746 {
2747 char dst_name[FN_REFLEN];
2748 ds_file_t *dstfile = NULL;
2749 xb_fil_cur_t cursor;
2750 xb_fil_cur_result_t res;
2751 xb_write_filt_ctxt_t write_filt_ctxt;
2752 const char *action;
2753 xb_read_filt_t *read_filter;
2754 my_bool rc = FALSE;
2755
2756 /* Get the name and the path for the tablespace. node->name always
2757 contains the path (which may be absolute for remote tablespaces in
2758 5.6+). space->name contains the tablespace name in the form
2759 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
2760 multi-node shared tablespace, space->name contains the name of the first
2761 node, but that's irrelevant, since we only need node_name to match them
2762 against filters, and the shared tablespace is always copied regardless
2763 of the filters value. */
2764
2765 const char* const node_name = node->space->name;
2766 const char* const node_path = node->name;
2767
2768 if (fil_is_user_tablespace_id(node->space->id)
2769 && check_if_skip_table(node_name)) {
2770 msg(thread_n, "Skipping %s.", node_name);
2771 return(FALSE);
2772 }
2773
2774 memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
2775
2776 bool was_dropped;
2777 pthread_mutex_lock(&backup_mutex);
2778 was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end());
2779 pthread_mutex_unlock(&backup_mutex);
2780 if (was_dropped) {
2781 if (node->is_open()) {
2782 mutex_enter(&fil_system.mutex);
2783 node->close();
2784 mutex_exit(&fil_system.mutex);
2785 }
2786 goto skip;
2787 }
2788
2789 if (!changed_page_bitmap) {
2790 read_filter = &rf_pass_through;
2791 }
2792 else {
2793 read_filter = &rf_bitmap;
2794 }
2795
2796 res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX);
2797 if (res == XB_FIL_CUR_SKIP) {
2798 goto skip;
2799 } else if (res == XB_FIL_CUR_ERROR) {
2800 goto error;
2801 }
2802
2803 strncpy(dst_name, dest_name ? dest_name : cursor.rel_path,
2804 sizeof dst_name - 1);
2805 dst_name[sizeof dst_name - 1] = '\0';
2806
2807 ut_a(write_filter.process != NULL);
2808
2809 if (write_filter.init != NULL &&
2810 !write_filter.init(&write_filt_ctxt, dst_name, &cursor,
2811 opt_log_innodb_page_corruption ? &corrupted_pages : NULL)) {
2812 msg (thread_n, "mariabackup: error: failed to initialize page write filter.");
2813 goto error;
2814 }
2815
2816 dstfile = ds_open(ds_data, dst_name, &cursor.statinfo);
2817 if (dstfile == NULL) {
2818 msg(thread_n,"mariabackup: error: can't open the destination stream for %s", dst_name);
2819 goto error;
2820 }
2821
2822 action = xb_get_copy_action();
2823
2824 if (xtrabackup_stream) {
2825 msg(thread_n, "%s %s", action, node_path);
2826 } else {
2827 msg(thread_n, "%s %s to %s", action, node_path, dstfile->path);
2828 }
2829
2830 /* The main copy loop */
2831 while ((res = xb_fil_cur_read(&cursor, corrupted_pages)) ==
2832 XB_FIL_CUR_SUCCESS) {
2833 if (!write_filter.process(&write_filt_ctxt, dstfile)) {
2834 goto error;
2835 }
2836 }
2837
2838 if (res == XB_FIL_CUR_ERROR) {
2839 goto error;
2840 }
2841
2842 if (write_filter.finalize
2843 && !write_filter.finalize(&write_filt_ctxt, dstfile)) {
2844 goto error;
2845 }
2846
2847 pthread_mutex_lock(&backup_mutex);
2848 ddl_tracker.tables_in_backup[node->space->id] = node_name;
2849 pthread_mutex_unlock(&backup_mutex);
2850
2851 /* close */
2852 msg(thread_n," ...done");
2853 xb_fil_cur_close(&cursor);
2854 if (ds_close(dstfile)) {
2855 rc = TRUE;
2856 }
2857 if (write_filter.deinit) {
2858 write_filter.deinit(&write_filt_ctxt);
2859 }
2860 return(rc);
2861
2862 error:
2863 xb_fil_cur_close(&cursor);
2864 if (dstfile != NULL) {
2865 ds_close(dstfile);
2866 }
2867 if (write_filter.deinit) {
2868 write_filter.deinit(&write_filt_ctxt);;
2869 }
2870 msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed.");
2871 return(TRUE); /*ERROR*/
2872
2873 skip:
2874
2875 if (dstfile != NULL) {
2876 ds_close(dstfile);
2877 }
2878 if (write_filter.deinit) {
2879 write_filter.deinit(&write_filt_ctxt);
2880 }
2881 msg(thread_n,"Warning: We assume the table was dropped during xtrabackup execution and ignore the tablespace %s", node_name);
2882 return(FALSE);
2883 }
2884
2885 /** Copy redo log blocks to the data sink.
2886 @param start_lsn buffer start LSN
2887 @param end_lsn buffer end LSN
2888 @param last whether we are copying the final part of the log
2889 @return last scanned LSN
2890 @retval 0 on failure */
xtrabackup_copy_log(lsn_t start_lsn,lsn_t end_lsn,bool last)2891 static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last)
2892 {
2893 lsn_t scanned_lsn = start_lsn;
2894 const byte* log_block = log_sys.buf;
2895 bool more_data = false;
2896
2897 for (ulint scanned_checkpoint = 0;
2898 scanned_lsn < end_lsn;
2899 log_block += OS_FILE_LOG_BLOCK_SIZE) {
2900 ulint checkpoint = log_block_get_checkpoint_no(log_block);
2901
2902 if (scanned_checkpoint > checkpoint
2903 && scanned_checkpoint - checkpoint >= 0x80000000UL) {
2904 /* Garbage from a log buffer flush which was made
2905 before the most recent database recovery */
2906 msg(0,"checkpoint wrap: " LSN_PF ",%zx,%zx",
2907 scanned_lsn, scanned_checkpoint, checkpoint);
2908 break;
2909 }
2910
2911 scanned_checkpoint = checkpoint;
2912
2913 ulint data_len = log_block_get_data_len(log_block);
2914
2915 more_data = recv_sys_add_to_parsing_buf(
2916 log_block,
2917 scanned_lsn + data_len);
2918
2919 recv_sys->scanned_lsn = scanned_lsn + data_len;
2920
2921 if (data_len == OS_FILE_LOG_BLOCK_SIZE) {
2922 /* We got a full log block. */
2923 scanned_lsn += data_len;
2924 } else if (data_len
2925 >= OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE
2926 || data_len < LOG_BLOCK_HDR_SIZE) {
2927 /* We got a garbage block (abrupt end of the log). */
2928 msg(0,"garbage block: " LSN_PF ",%zu",scanned_lsn, data_len);
2929 break;
2930 } else {
2931 /* We got a partial block (abrupt end of the log). */
2932 scanned_lsn += data_len;
2933 break;
2934 }
2935 }
2936
2937 store_t store = STORE_NO;
2938
2939 if (more_data && recv_parse_log_recs(0, &store, 0, false)) {
2940
2941 msg("Error: copying the log failed");
2942
2943 return(0);
2944 }
2945
2946 recv_sys_justify_left_parsing_buf();
2947
2948 log_sys.log.scanned_lsn = scanned_lsn;
2949
2950 end_lsn = last
2951 ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE)
2952 : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1);
2953
2954 if (ulint write_size = ulint(end_lsn - start_lsn)) {
2955 if (srv_encrypt_log) {
2956 log_crypt(log_sys.buf, start_lsn, write_size);
2957 }
2958
2959 if (ds_write(dst_log_file, log_sys.buf, write_size)) {
2960 msg("Error: write to logfile failed");
2961 return(0);
2962 }
2963 }
2964
2965 return(scanned_lsn);
2966 }
2967
2968 /** Copy redo log until the current end of the log is reached
2969 @param last whether we are copying the final part of the log
2970 @return whether the operation failed */
xtrabackup_copy_logfile(bool last=false)2971 static bool xtrabackup_copy_logfile(bool last = false)
2972 {
2973 ut_a(dst_log_file != NULL);
2974 ut_ad(recv_sys != NULL);
2975
2976 bool overwritten_block = false;
2977 lsn_t start_lsn;
2978 lsn_t end_lsn;
2979
2980 recv_sys->parse_start_lsn = log_copy_scanned_lsn;
2981 recv_sys->scanned_lsn = log_copy_scanned_lsn;
2982
2983 start_lsn = ut_uint64_align_down(log_copy_scanned_lsn,
2984 OS_FILE_LOG_BLOCK_SIZE);
2985 do {
2986 end_lsn = start_lsn + RECV_SCAN_SIZE;
2987
2988 xtrabackup_io_throttling();
2989
2990 log_mutex_enter();
2991 lsn_t lsn= start_lsn;
2992 for (int retries= 0; retries < 100; retries++) {
2993 if (log_sys.log.read_log_seg(&lsn, end_lsn)
2994 || lsn != start_lsn) {
2995 break;
2996 }
2997 msg("Retrying read of log at LSN=" LSN_PF, lsn);
2998 my_sleep(1000);
2999 }
3000
3001 if (lsn == start_lsn) {
3002 overwritten_block= !recv_sys->found_corrupt_log
3003 && (innodb_log_checksums || log_sys.log.is_encrypted())
3004 && log_block_calc_checksum_crc32(log_sys.buf) ==
3005 log_block_get_checksum(log_sys.buf)
3006 && log_block_get_hdr_no(log_sys.buf) >
3007 log_block_convert_lsn_to_no(start_lsn);
3008 start_lsn = 0;
3009 } else {
3010 mutex_enter(&recv_sys->mutex);
3011 start_lsn = xtrabackup_copy_log(start_lsn, lsn, last);
3012 mutex_exit(&recv_sys->mutex);
3013 }
3014
3015 log_mutex_exit();
3016
3017 if (!start_lsn) {
3018 const char *reason = recv_sys->found_corrupt_log
3019 ? "corrupt log."
3020 : (overwritten_block
3021 ? "redo log block is overwritten, please increase redo log size with innodb_log_file_size parameter."
3022 : ((innodb_log_checksums || log_sys.log.is_encrypted())
3023 ? "redo log block checksum does not match."
3024 : "unknown reason as innodb_log_checksums is switched off and redo"
3025 " log is not encrypted."));
3026 die("xtrabackup_copy_logfile() failed: %s", reason);
3027 return true;
3028 }
3029 } while (start_lsn == end_lsn);
3030
3031 ut_ad(start_lsn == log_sys.log.scanned_lsn);
3032
3033 msg(">> log scanned up to (" LSN_PF ")", start_lsn);
3034
3035 /* update global variable*/
3036 pthread_mutex_lock(&backup_mutex);
3037 log_copy_scanned_lsn = start_lsn;
3038 pthread_cond_broadcast(&scanned_lsn_cond);
3039 pthread_mutex_unlock(&backup_mutex);
3040 return(false);
3041 }
3042
3043 /**
3044 Wait until redo log copying thread processes given lsn
3045 */
backup_wait_for_lsn(lsn_t lsn)3046 void backup_wait_for_lsn(lsn_t lsn) {
3047 bool completed = false;
3048 pthread_mutex_lock(&backup_mutex);
3049 do {
3050 pthread_cond_wait(&scanned_lsn_cond, &backup_mutex);
3051 completed = log_copy_scanned_lsn >= lsn;
3052 } while (!completed);
3053 pthread_mutex_unlock(&backup_mutex);
3054 }
3055
3056 extern lsn_t server_lsn_after_lock;
3057
DECLARE_THREAD(log_copying_thread)3058 static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
3059 {
3060 /*
3061 Initialize mysys thread-specific memory so we can
3062 use mysys functions in this thread.
3063 */
3064 my_thread_init();
3065
3066 for (;;) {
3067 os_event_reset(log_copying_stop);
3068 os_event_wait_time_low(log_copying_stop,
3069 xtrabackup_log_copy_interval * 1000ULL,
3070 0);
3071 if (xtrabackup_copy_logfile()) {
3072 break;
3073 }
3074
3075 log_mutex_enter();
3076 bool completed = metadata_to_lsn
3077 && metadata_to_lsn <= log_copy_scanned_lsn;
3078 log_mutex_exit();
3079 if (completed) {
3080 break;
3081 }
3082 }
3083
3084 log_copying_running = false;
3085 my_thread_end();
3086 os_thread_exit();
3087
3088 return(0);
3089 }
3090
3091 /* io throttle watching (rough) */
DECLARE_THREAD(io_watching_thread)3092 static os_thread_ret_t DECLARE_THREAD(io_watching_thread)(void*)
3093 {
3094 /* currently, for --backup only */
3095 ut_a(xtrabackup_backup);
3096
3097 while (log_copying_running && !metadata_to_lsn) {
3098 os_thread_sleep(1000000); /*1 sec*/
3099 io_ticket = xtrabackup_throttle;
3100 os_event_set(wait_throttle);
3101 }
3102
3103 /* stop io throttle */
3104 xtrabackup_throttle = 0;
3105 os_event_set(wait_throttle);
3106
3107 io_watching_thread_running = false;
3108
3109 os_thread_exit();
3110
3111 return(0);
3112 }
3113
3114 #ifndef DBUG_OFF
dbug_mariabackup_get_val(const char * event,const char * key)3115 char *dbug_mariabackup_get_val(const char *event, const char *key)
3116 {
3117 char envvar[FN_REFLEN];
3118 if (key) {
3119 snprintf(envvar, sizeof(envvar), "%s_%s", event, key);
3120 char *slash = strchr(envvar, '/');
3121 if (slash)
3122 *slash = '_';
3123 } else {
3124 strncpy(envvar, event, sizeof envvar - 1);
3125 envvar[sizeof envvar - 1] = '\0';
3126 }
3127 return getenv(envvar);
3128 }
3129
3130 /*
3131 In debug mode, execute SQL statement that was passed via environment.
3132 To use this facility, you need to
3133
3134 1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key););
3135 to the code. key is usually a table name
3136 2. Set environment variable my_event_name_$key SQL statement you want to execute
3137 when event occurs, in DBUG_EXECUTE_IF from above.
3138 In mtr , you can set environment via 'let' statement (do not use $ as the first char
3139 for the variable)
3140 3. start mariabackup with --dbug=+d,debug_mariabackup_events
3141 */
dbug_mariabackup_event(const char * event,const char * key)3142 void dbug_mariabackup_event(const char *event,const char *key)
3143 {
3144 char *sql = dbug_mariabackup_get_val(event, key);
3145 if (sql && *sql) {
3146 msg("dbug_mariabackup_event : executing '%s'", sql);
3147 xb_mysql_query(mysql_connection, sql, false, true);
3148 }
3149 }
3150 #endif // DBUG_OFF
3151
3152 /**************************************************************************
3153 Datafiles copying thread.*/
3154 static
3155 os_thread_ret_t
DECLARE_THREAD(data_copy_thread_func)3156 DECLARE_THREAD(data_copy_thread_func)(
3157 /*==================*/
3158 void *arg) /* thread context */
3159 {
3160 data_thread_ctxt_t *ctxt = (data_thread_ctxt_t *) arg;
3161 uint num = ctxt->num;
3162 fil_node_t* node;
3163 ut_ad(ctxt->corrupted_pages);
3164
3165 /*
3166 Initialize mysys thread-specific memory so we can
3167 use mysys functions in this thread.
3168 */
3169 my_thread_init();
3170
3171 while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
3172 DBUG_MARIABACKUP_EVENT("before_copy", node->space->name);
3173 DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", node->space->name,
3174 backup_wait_for_lsn(get_current_lsn(mysql_connection)););
3175 /* copy the datafile */
3176 if (xtrabackup_copy_datafile(node, num, NULL,
3177 xtrabackup_incremental ? wf_incremental : wf_write_through,
3178 *ctxt->corrupted_pages))
3179 die("failed to copy datafile.");
3180
3181 DBUG_MARIABACKUP_EVENT("after_copy", node->space->name);
3182
3183 }
3184
3185 pthread_mutex_lock(ctxt->count_mutex);
3186 (*ctxt->count)--;
3187 pthread_mutex_unlock(ctxt->count_mutex);
3188
3189 my_thread_end();
3190 os_thread_exit();
3191 OS_THREAD_DUMMY_RETURN;
3192 }
3193
3194 /************************************************************************
3195 Initialize the appropriate datasink(s). Both local backups and streaming in the
3196 'xbstream' format allow parallel writes so we can write directly.
3197
3198 Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks
3199 for the data stream (and don't allow parallel data copying) and for metainfo
3200 files (including ib_logfile0). The second datasink writes to temporary
3201 files first, and then streams them in a serialized way when closed. */
3202 static void
xtrabackup_init_datasinks(void)3203 xtrabackup_init_datasinks(void)
3204 {
3205 /* Start building out the pipelines from the terminus back */
3206 if (xtrabackup_stream) {
3207 /* All streaming goes to stdout */
3208 ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
3209 DS_TYPE_STDOUT);
3210 } else {
3211 /* Local filesystem */
3212 ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
3213 DS_TYPE_LOCAL);
3214 }
3215
3216 /* Track it for destruction */
3217 xtrabackup_add_datasink(ds_data);
3218
3219 /* Stream formatting */
3220 if (xtrabackup_stream) {
3221 ds_ctxt_t *ds;
3222
3223 ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM);
3224 ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM);
3225
3226 xtrabackup_add_datasink(ds);
3227
3228 ds_set_pipe(ds, ds_data);
3229 ds_data = ds;
3230
3231
3232 ds_redo = ds_meta = ds_data;
3233 }
3234
3235 /* Compression for ds_data and ds_redo */
3236 if (xtrabackup_compress) {
3237 ds_ctxt_t *ds;
3238
3239 /* Use a 1 MB buffer for compressed output stream */
3240 ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
3241 ds_buffer_set_size(ds, 1024 * 1024);
3242 xtrabackup_add_datasink(ds);
3243 ds_set_pipe(ds, ds_data);
3244 if (ds_data != ds_redo) {
3245 ds_data = ds;
3246 ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
3247 ds_buffer_set_size(ds, 1024 * 1024);
3248 xtrabackup_add_datasink(ds);
3249 ds_set_pipe(ds, ds_redo);
3250 ds_redo = ds;
3251 } else {
3252 ds_redo = ds_data = ds;
3253 }
3254
3255 ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
3256 xtrabackup_add_datasink(ds);
3257 ds_set_pipe(ds, ds_data);
3258 if (ds_data != ds_redo) {
3259 ds_data = ds;
3260 ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
3261 xtrabackup_add_datasink(ds);
3262 ds_set_pipe(ds, ds_redo);
3263 ds_redo = ds;
3264 } else {
3265 ds_redo = ds_data = ds;
3266 }
3267 }
3268 }
3269
3270 /************************************************************************
3271 Destroy datasinks.
3272
3273 Destruction is done in the specific order to not violate their order in the
3274 pipeline so that each datasink is able to flush data down the pipeline. */
xtrabackup_destroy_datasinks(void)3275 static void xtrabackup_destroy_datasinks(void)
3276 {
3277 for (uint i = actual_datasinks; i > 0; i--) {
3278 ds_destroy(datasinks[i-1]);
3279 datasinks[i-1] = NULL;
3280 }
3281 ds_data = NULL;
3282 ds_meta = NULL;
3283 ds_redo = NULL;
3284 }
3285
3286 #define SRV_MAX_N_PENDING_SYNC_IOS 100
3287
3288 /** Initialize the tablespace cache subsystem. */
3289 static
3290 void
xb_fil_io_init()3291 xb_fil_io_init()
3292 {
3293 fil_system.create(srv_file_per_table ? 50000 : 5000);
3294 }
3295
3296 static
3297 Datafile*
xb_new_datafile(const char * name,bool is_remote)3298 xb_new_datafile(const char *name, bool is_remote)
3299 {
3300 if (is_remote) {
3301 RemoteDatafile *remote_file = new RemoteDatafile();
3302 remote_file->set_name(name);
3303 return(remote_file);
3304 } else {
3305 Datafile *file = new Datafile();
3306 file->set_name(name);
3307 file->make_filepath(".", name, IBD);
3308 return(file);
3309 }
3310 }
3311
3312
3313 /** Load tablespace.
3314
3315 @param[in] dirname directory name of the tablespace to open
3316 @param[in] filname file name of the tablespece to open
3317 @param[in] is_remote true if tablespace file is .isl
3318 @param[in] set_size true if we need to set tablespace size in pages explixitly.
3319 If this parameter is set, the size and free pages limit will not be read
3320 from page 0.
3321 */
xb_load_single_table_tablespace(const char * dirname,const char * filname,bool is_remote,bool set_size)3322 static void xb_load_single_table_tablespace(const char *dirname,
3323 const char *filname,
3324 bool is_remote, bool set_size)
3325 {
3326 ut_ad(srv_operation == SRV_OPERATION_BACKUP
3327 || srv_operation == SRV_OPERATION_RESTORE_DELTA
3328 || srv_operation == SRV_OPERATION_RESTORE);
3329 /* Ignore .isl files on XtraBackup recovery. All tablespaces must be
3330 local. */
3331 if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) {
3332 return;
3333 }
3334 if (check_if_skip_table(filname)) {
3335 return;
3336 }
3337
3338 /* The name ends in .ibd or .isl;
3339 try opening the file */
3340 char* name;
3341 size_t dirlen = dirname == NULL ? 0 : strlen(dirname);
3342 size_t namelen = strlen(filname);
3343 ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2;
3344 lsn_t flush_lsn;
3345 dberr_t err;
3346 fil_space_t *space;
3347
3348 name = static_cast<char*>(ut_malloc_nokey(pathlen));
3349
3350 if (dirname != NULL) {
3351 snprintf(name, pathlen, "%s/%s", dirname, filname);
3352 name[pathlen - 5] = 0;
3353 } else {
3354 snprintf(name, pathlen, "%s", filname);
3355 name[pathlen - 5] = 0;
3356 }
3357
3358 Datafile *file = xb_new_datafile(name, is_remote);
3359
3360 if (file->open_read_only(true) != DB_SUCCESS) {
3361 die("Can't open datafile %s", name);
3362 }
3363
3364 for (int i = 0; i < 10; i++) {
3365 err = file->validate_first_page(&flush_lsn);
3366 if (err != DB_CORRUPTION) {
3367 break;
3368 }
3369
3370 my_sleep(1000);
3371 }
3372
3373 bool is_empty_file = file->exists() && file->is_empty_file();
3374
3375 if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
3376 os_offset_t n_pages = 0;
3377 if (set_size) {
3378 os_offset_t node_size = os_file_get_size(file->handle());
3379 ut_a(node_size != (os_offset_t) -1);
3380 n_pages = node_size / page_size_t(file->flags()).physical();
3381 }
3382 space = fil_space_create(
3383 name, file->space_id(), file->flags(),
3384 FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */);
3385
3386 ut_a(space != NULL);
3387
3388 space->add(file->filepath(), OS_FILE_CLOSED, ulint(n_pages),
3389 false, false);
3390 /* by opening the tablespace we forcing node and space objects
3391 in the cache to be populated with fields from space header */
3392 space->open();
3393
3394 if (srv_operation == SRV_OPERATION_RESTORE_DELTA
3395 || xb_close_files) {
3396 space->close();
3397 }
3398 }
3399
3400 delete file;
3401
3402 if (err != DB_SUCCESS && xtrabackup_backup && !is_empty_file) {
3403 die("Failed to validate first page of the file %s, error %d",name, (int)err);
3404 }
3405
3406 ut_free(name);
3407 }
3408
xb_load_single_table_tablespace(const std::string & space_name,bool set_size)3409 static void xb_load_single_table_tablespace(const std::string &space_name,
3410 bool set_size)
3411 {
3412 std::string name(space_name);
3413 bool is_remote= access((name + ".ibd").c_str(), R_OK) != 0;
3414 const char *extension= is_remote ? ".isl" : ".ibd";
3415 name.append(extension);
3416 char buf[FN_REFLEN];
3417 strncpy(buf, name.c_str(), sizeof buf - 1);
3418 buf[sizeof buf - 1]= '\0';
3419 const char *dbname= buf;
3420 char *p= strchr(buf, '/');
3421 if (p == 0)
3422 die("Unexpected tablespace %s filename %s", space_name.c_str(),
3423 name.c_str());
3424 ut_a(p);
3425 *p= 0;
3426 const char *tablename= p + 1;
3427 xb_load_single_table_tablespace(dbname, tablename, is_remote, set_size);
3428 }
3429
3430 /** Scan the database directories under the MySQL datadir, looking for
3431 .ibd files and determining the space id in each of them.
3432 @return DB_SUCCESS or error number */
3433
enumerate_ibd_files(process_single_tablespace_func_t callback)3434 static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback)
3435 {
3436 int ret;
3437 char* dbpath = NULL;
3438 ulint dbpath_len = 100;
3439 os_file_dir_t dir;
3440 os_file_dir_t dbdir;
3441 os_file_stat_t dbinfo;
3442 os_file_stat_t fileinfo;
3443 dberr_t err = DB_SUCCESS;
3444 size_t len;
3445
3446 /* The datadir of MySQL is always the default directory of mysqld */
3447
3448 dir = os_file_opendir(fil_path_to_mysql_datadir, true);
3449
3450 if (dir == NULL) {
3451
3452 return(DB_ERROR);
3453 }
3454
3455 dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
3456
3457 /* Scan all directories under the datadir. They are the database
3458 directories of MySQL. */
3459
3460 ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir,
3461 &dbinfo);
3462 while (ret == 0) {
3463
3464 /* General tablespaces are always at the first level of the
3465 data home dir */
3466 if (dbinfo.type == OS_FILE_TYPE_FILE) {
3467 bool is_isl = ends_with(dbinfo.name, ".isl");
3468 bool is_ibd = !is_isl && ends_with(dbinfo.name,".ibd");
3469
3470 if (is_isl || is_ibd) {
3471 (*callback)(NULL, dbinfo.name, is_isl, false);
3472 }
3473 }
3474
3475 if (dbinfo.type == OS_FILE_TYPE_FILE
3476 || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
3477
3478 goto next_datadir_item;
3479 }
3480
3481 /* We found a symlink or a directory; try opening it to see
3482 if a symlink is a directory */
3483
3484 len = strlen(fil_path_to_mysql_datadir)
3485 + strlen (dbinfo.name) + 2;
3486 if (len > dbpath_len) {
3487 dbpath_len = len;
3488
3489 if (dbpath) {
3490 ut_free(dbpath);
3491 }
3492
3493 dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
3494 }
3495 snprintf(dbpath, dbpath_len,
3496 "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
3497 os_normalize_path(dbpath);
3498
3499 if (check_if_skip_database_by_path(dbpath)) {
3500 fprintf(stderr, "Skipping db: %s\n", dbpath);
3501 goto next_datadir_item;
3502 }
3503
3504 /* We want wrong directory permissions to be a fatal error for
3505 XtraBackup. */
3506 dbdir = os_file_opendir(dbpath, true);
3507
3508 if (dbdir != NULL) {
3509
3510 /* We found a database directory; loop through it,
3511 looking for possible .ibd files in it */
3512
3513 for (ret = fil_file_readdir_next_file(&err, dbpath,
3514 dbdir,
3515 &fileinfo);
3516 ret == 0;
3517 ret = fil_file_readdir_next_file(&err, dbpath,
3518 dbdir,
3519 &fileinfo)) {
3520 if (fileinfo.type == OS_FILE_TYPE_DIR) {
3521 continue;
3522 }
3523
3524 /* We found a symlink or a file */
3525 if (strlen(fileinfo.name) > 4) {
3526 bool is_isl= false;
3527 if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".isl"))))
3528 (*callback)(dbinfo.name, fileinfo.name, is_isl, false);
3529 }
3530 }
3531
3532 if (0 != os_file_closedir(dbdir)) {
3533 fprintf(stderr, "InnoDB: Warning: could not"
3534 " close database directory %s\n",
3535 dbpath);
3536
3537 err = DB_ERROR;
3538 }
3539
3540 } else {
3541
3542 err = DB_ERROR;
3543 break;
3544
3545 }
3546
3547 next_datadir_item:
3548 ret = fil_file_readdir_next_file(&err,
3549 fil_path_to_mysql_datadir,
3550 dir, &dbinfo);
3551 }
3552
3553 ut_free(dbpath);
3554
3555 if (0 != os_file_closedir(dir)) {
3556 fprintf(stderr,
3557 "InnoDB: Error: could not close MySQL datadir\n");
3558
3559 return(DB_ERROR);
3560 }
3561
3562 return(err);
3563 }
3564
3565 /** Assign srv_undo_space_id_start variable if there are undo tablespace present.
3566 Read the TRX_SYS page from ibdata1 file and get the minimum space id from
3567 the first slot rollback segments of TRX_SYS_PAGE_NO.
3568 @retval DB_ERROR if file open or page read failed.
3569 @retval DB_SUCCESS if srv_undo_space_id assigned successfully. */
xb_assign_undo_space_start()3570 static dberr_t xb_assign_undo_space_start()
3571 {
3572
3573 pfs_os_file_t file;
3574 byte* buf;
3575 byte* page;
3576 bool ret;
3577 dberr_t error = DB_SUCCESS;
3578 ulint space;
3579 int n_retries = 5;
3580
3581 if (srv_undo_tablespaces == 0) {
3582 return error;
3583 }
3584
3585 file = os_file_create(0, srv_sys_space.first_datafile()->filepath(),
3586 OS_FILE_OPEN, OS_FILE_NORMAL, OS_DATA_FILE, true, &ret);
3587
3588 if (!ret) {
3589 msg("Error opening %s", srv_sys_space.first_datafile()->filepath());
3590 return DB_ERROR;
3591 }
3592
3593 buf = static_cast<byte*>(ut_malloc_nokey(2U << srv_page_size_shift));
3594 page = static_cast<byte*>(ut_align(buf, srv_page_size));
3595
3596 retry:
3597 if (os_file_read(IORequestRead, file, page,
3598 TRX_SYS_PAGE_NO << srv_page_size_shift,
3599 srv_page_size) != DB_SUCCESS) {
3600 msg("Reading TRX_SYS page failed.");
3601 error = DB_ERROR;
3602 goto func_exit;
3603 }
3604
3605 /* TRX_SYS page can't be compressed or encrypted. */
3606 if (buf_page_is_corrupted(false, page, univ_page_size)) {
3607 if (n_retries--) {
3608 os_thread_sleep(1000);
3609 goto retry;
3610 } else {
3611 msg("mariabackup: TRX_SYS page corrupted.\n");
3612 error = DB_ERROR;
3613 goto func_exit;
3614 }
3615 }
3616
3617 /* 0th slot always points to system tablespace.
3618 1st slot should point to first undotablespace which is minimum. */
3619
3620 ut_ad(mach_read_from_4(TRX_SYS + TRX_SYS_RSEGS
3621 + TRX_SYS_RSEG_SLOT_SIZE
3622 + TRX_SYS_RSEG_PAGE_NO + page)
3623 != FIL_NULL);
3624
3625 space = mach_read_ulint(TRX_SYS + TRX_SYS_RSEGS
3626 + TRX_SYS_RSEG_SLOT_SIZE
3627 + TRX_SYS_RSEG_SPACE + page, MLOG_4BYTES);
3628
3629 srv_undo_space_id_start = space;
3630
3631 func_exit:
3632 ut_free(buf);
3633 ret = os_file_close(file);
3634 ut_a(ret);
3635
3636 return error;
3637 }
3638
3639 /****************************************************************************
3640 Populates the tablespace memory cache by scanning for and opening data files.
3641 @returns DB_SUCCESS or error code.*/
3642 static
3643 dberr_t
xb_load_tablespaces()3644 xb_load_tablespaces()
3645 {
3646 bool create_new_db;
3647 dberr_t err;
3648 ulint sum_of_new_sizes;
3649 lsn_t flush_lsn;
3650
3651 ut_ad(srv_operation == SRV_OPERATION_BACKUP
3652 || srv_operation == SRV_OPERATION_RESTORE_DELTA);
3653
3654 err = srv_sys_space.check_file_spec(&create_new_db, 0);
3655
3656 /* create_new_db must not be true. */
3657 if (err != DB_SUCCESS || create_new_db) {
3658 msg("Could not find data files at the specified datadir");
3659 return(DB_ERROR);
3660 }
3661
3662 for (int i= 0; i < 10; i++) {
3663 err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes,
3664 &flush_lsn);
3665 if (err == DB_PAGE_CORRUPTED || err == DB_CORRUPTION) {
3666 my_sleep(1000);
3667 }
3668 else
3669 break;
3670 }
3671
3672 if (err != DB_SUCCESS) {
3673 msg("Could not open data files.\n");
3674 return(err);
3675 }
3676
3677 /* Add separate undo tablespaces to fil_system */
3678
3679 err = xb_assign_undo_space_start();
3680
3681 if (err != DB_SUCCESS) {
3682 return err;
3683 }
3684
3685 err = srv_undo_tablespaces_init(false);
3686
3687 if (err != DB_SUCCESS) {
3688 return(err);
3689 }
3690
3691 /* It is important to call xb_load_single_table_tablespaces() after
3692 srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() *
3693 relies on srv_undo_tablespaces_open to be properly initialized */
3694
3695 msg("mariabackup: Generating a list of tablespaces");
3696
3697 err = enumerate_ibd_files(xb_load_single_table_tablespace);
3698 if (err != DB_SUCCESS) {
3699 return(err);
3700 }
3701 DBUG_MARIABACKUP_EVENT("after_load_tablespaces", 0);
3702 return(DB_SUCCESS);
3703 }
3704
3705 /************************************************************************
3706 Initialize the tablespace memory cache and populate it by scanning for and
3707 opening data files.
3708 @returns DB_SUCCESS or error code.*/
3709 static
3710 dberr_t
xb_data_files_init()3711 xb_data_files_init()
3712 {
3713 xb_fil_io_init();
3714
3715 return(xb_load_tablespaces());
3716 }
3717
3718 /************************************************************************
3719 Destroy the tablespace memory cache. */
3720 static
3721 void
xb_data_files_close()3722 xb_data_files_close()
3723 {
3724 ut_ad(!os_thread_count);
3725 fil_close_all_files();
3726 if (buf_dblwr) {
3727 buf_dblwr_free();
3728 }
3729 }
3730
3731 /***********************************************************************
3732 Allocate and initialize the entry for databases and tables filtering
3733 hash tables. If memory allocation is not successful, terminate program.
3734 @return pointer to the created entry. */
3735 static
3736 xb_filter_entry_t *
xb_new_filter_entry(const char * name)3737 xb_new_filter_entry(
3738 /*================*/
3739 const char* name) /*!< in: name of table/database */
3740 {
3741 xb_filter_entry_t *entry;
3742 ulint namelen = strlen(name);
3743
3744 ut_a(namelen <= NAME_LEN * 2 + 1);
3745
3746 entry = static_cast<xb_filter_entry_t *>
3747 (malloc(sizeof(xb_filter_entry_t) + namelen + 1));
3748 memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1);
3749 entry->name = ((char*)entry) + sizeof(xb_filter_entry_t);
3750 strcpy(entry->name, name);
3751 entry->has_tables = FALSE;
3752
3753 return entry;
3754 }
3755
3756 /***********************************************************************
3757 Add entry to hash table. If hash table is NULL, allocate and initialize
3758 new hash table */
3759 static
3760 xb_filter_entry_t*
xb_add_filter(const char * name,hash_table_t ** hash)3761 xb_add_filter(
3762 /*========================*/
3763 const char* name, /*!< in: name of table/database */
3764 hash_table_t** hash) /*!< in/out: hash to insert into */
3765 {
3766 xb_filter_entry_t* entry;
3767
3768 entry = xb_new_filter_entry(name);
3769
3770 if (UNIV_UNLIKELY(*hash == NULL)) {
3771 *hash = hash_create(1000);
3772 }
3773 HASH_INSERT(xb_filter_entry_t,
3774 name_hash, *hash,
3775 ut_fold_string(entry->name),
3776 entry);
3777
3778 return entry;
3779 }
3780
3781 /***********************************************************************
3782 Validate name of table or database. If name is invalid, program will
3783 be finished with error code */
3784 static
3785 void
xb_validate_name(const char * name,size_t len)3786 xb_validate_name(
3787 /*=============*/
3788 const char* name, /*!< in: name */
3789 size_t len) /*!< in: length of name */
3790 {
3791 const char* p;
3792
3793 /* perform only basic validation. validate length and
3794 path symbols */
3795 if (len > NAME_LEN) {
3796 die("name `%s` is too long.", name);
3797 }
3798 p = strpbrk(name, "/\\~");
3799 if (p && (uint) (p - name) < NAME_LEN) {
3800 die("name `%s` is not valid.", name);
3801 }
3802 }
3803
3804 /***********************************************************************
3805 Register new filter entry which can be either database
3806 or table name. */
3807 static
3808 void
xb_register_filter_entry(const char * name,hash_table_t ** databases_hash,hash_table_t ** tables_hash)3809 xb_register_filter_entry(
3810 /*=====================*/
3811 const char* name, /*!< in: name */
3812 hash_table_t** databases_hash,
3813 hash_table_t** tables_hash
3814 )
3815 {
3816 const char* p;
3817 size_t namelen;
3818 xb_filter_entry_t* db_entry = NULL;
3819
3820 namelen = strlen(name);
3821 if ((p = strchr(name, '.')) != NULL) {
3822 char dbname[NAME_LEN + 1];
3823
3824 xb_validate_name(name, p - name);
3825 xb_validate_name(p + 1, namelen - (p - name));
3826
3827 strncpy(dbname, name, p - name);
3828 dbname[p - name] = 0;
3829
3830 if (*databases_hash) {
3831 HASH_SEARCH(name_hash, (*databases_hash),
3832 ut_fold_string(dbname),
3833 xb_filter_entry_t*,
3834 db_entry, (void) 0,
3835 !strcmp(db_entry->name, dbname));
3836 }
3837 if (!db_entry) {
3838 db_entry = xb_add_filter(dbname, databases_hash);
3839 }
3840 db_entry->has_tables = TRUE;
3841 xb_add_filter(name, tables_hash);
3842 } else {
3843 xb_validate_name(name, namelen);
3844
3845 xb_add_filter(name, databases_hash);
3846 }
3847 }
3848
3849 static
3850 void
xb_register_include_filter_entry(const char * name)3851 xb_register_include_filter_entry(
3852 const char* name
3853 )
3854 {
3855 xb_register_filter_entry(name, &databases_include_hash,
3856 &tables_include_hash);
3857 }
3858
3859 static
3860 void
xb_register_exclude_filter_entry(const char * name)3861 xb_register_exclude_filter_entry(
3862 const char* name
3863 )
3864 {
3865 xb_register_filter_entry(name, &databases_exclude_hash,
3866 &tables_exclude_hash);
3867 }
3868
register_ignore_db_dirs_filter(const char * name)3869 void register_ignore_db_dirs_filter(const char *name)
3870 {
3871 xb_add_filter(name, &databases_exclude_hash);
3872 }
3873
3874 /***********************************************************************
3875 Register new table for the filter. */
3876 static
3877 void
xb_register_table(const char * name)3878 xb_register_table(
3879 /*==============*/
3880 const char* name) /*!< in: name of table */
3881 {
3882 if (strchr(name, '.') == NULL) {
3883 die("`%s` is not fully qualified name.", name);
3884 }
3885
3886 xb_register_include_filter_entry(name);
3887 }
3888
3889 static
3890 void
xb_add_regex_to_list(const char * regex,const char * error_context,regex_list_t * list)3891 xb_add_regex_to_list(
3892 const char* regex, /*!< in: regex */
3893 const char* error_context, /*!< in: context to error message */
3894 regex_list_t* list) /*! in: list to put new regex to */
3895 {
3896 char errbuf[100];
3897 int ret;
3898
3899 regex_t compiled_regex;
3900 ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
3901
3902 if (ret != 0) {
3903 regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
3904 msg("mariabackup: error: %s regcomp(%s): %s",
3905 error_context, regex, errbuf);
3906 exit(EXIT_FAILURE);
3907 }
3908
3909 list->push_back(compiled_regex);
3910 }
3911
3912 /***********************************************************************
3913 Register new regex for the include filter. */
3914 static
3915 void
xb_register_include_regex(const char * regex)3916 xb_register_include_regex(
3917 /*==============*/
3918 const char* regex) /*!< in: regex */
3919 {
3920 xb_add_regex_to_list(regex, "tables", ®ex_include_list);
3921 }
3922
3923 /***********************************************************************
3924 Register new regex for the exclude filter. */
3925 static
3926 void
xb_register_exclude_regex(const char * regex)3927 xb_register_exclude_regex(
3928 /*==============*/
3929 const char* regex) /*!< in: regex */
3930 {
3931 xb_add_regex_to_list(regex, "tables-exclude", ®ex_exclude_list);
3932 }
3933
3934 typedef void (*insert_entry_func_t)(const char*);
3935
3936 /* Scan string and load filter entries from it.
3937 @param[in] list string representing a list
3938 @param[in] delimiters delimiters of entries
3939 @param[in] ins callback to add entry */
xb_load_list_string(char * list,const char * delimiters,insert_entry_func_t ins)3940 void xb_load_list_string(char *list, const char *delimiters,
3941 insert_entry_func_t ins)
3942 {
3943 char *p;
3944 char *saveptr;
3945
3946 p= strtok_r(list, delimiters, &saveptr);
3947 while (p)
3948 {
3949
3950 ins(p);
3951
3952 p= strtok_r(NULL, delimiters, &saveptr);
3953 }
3954 }
3955
3956 /***********************************************************************
3957 Scan file and load filter entries from it. */
3958 static
3959 void
xb_load_list_file(const char * filename,insert_entry_func_t ins)3960 xb_load_list_file(
3961 /*==============*/
3962 const char* filename, /*!< in: name of file */
3963 insert_entry_func_t ins) /*!< in: callback to add entry */
3964 {
3965 char name_buf[NAME_LEN*2+2];
3966 FILE* fp;
3967
3968 /* read and store the filenames */
3969 fp = fopen(filename, "r");
3970 if (!fp) {
3971 die("Can't open %s",
3972 filename);
3973 }
3974 while (fgets(name_buf, sizeof(name_buf), fp) != NULL) {
3975 char* p = strchr(name_buf, '\n');
3976 if (p) {
3977 *p = '\0';
3978 } else {
3979 die("`%s...` name is too long", name_buf);
3980 }
3981
3982 ins(name_buf);
3983 }
3984
3985 fclose(fp);
3986 }
3987
3988
3989 static
3990 void
xb_filters_init()3991 xb_filters_init()
3992 {
3993 if (xtrabackup_databases) {
3994 xb_load_list_string(xtrabackup_databases, " \t",
3995 xb_register_include_filter_entry);
3996 }
3997
3998 if (xtrabackup_databases_file) {
3999 xb_load_list_file(xtrabackup_databases_file,
4000 xb_register_include_filter_entry);
4001 }
4002
4003 if (xtrabackup_databases_exclude) {
4004 xb_load_list_string(xtrabackup_databases_exclude, " \t",
4005 xb_register_exclude_filter_entry);
4006 }
4007
4008 if (xtrabackup_tables) {
4009 xb_load_list_string(xtrabackup_tables, ",",
4010 xb_register_include_regex);
4011 }
4012
4013 if (xtrabackup_tables_file) {
4014 xb_load_list_file(xtrabackup_tables_file, xb_register_table);
4015 }
4016
4017 if (xtrabackup_tables_exclude) {
4018 xb_load_list_string(xtrabackup_tables_exclude, ",",
4019 xb_register_exclude_regex);
4020 }
4021 }
4022
4023 static
4024 void
xb_filter_hash_free(hash_table_t * hash)4025 xb_filter_hash_free(hash_table_t* hash)
4026 {
4027 ulint i;
4028
4029 /* free the hash elements */
4030 for (i = 0; i < hash_get_n_cells(hash); i++) {
4031 xb_filter_entry_t* table;
4032
4033 table = static_cast<xb_filter_entry_t *>
4034 (HASH_GET_FIRST(hash, i));
4035
4036 while (table) {
4037 xb_filter_entry_t* prev_table = table;
4038
4039 table = static_cast<xb_filter_entry_t *>
4040 (HASH_GET_NEXT(name_hash, prev_table));
4041
4042 HASH_DELETE(xb_filter_entry_t, name_hash, hash,
4043 ut_fold_string(prev_table->name), prev_table);
4044 free(prev_table);
4045 }
4046 }
4047
4048 /* free hash */
4049 hash_table_free(hash);
4050 }
4051
xb_regex_list_free(regex_list_t * list)4052 static void xb_regex_list_free(regex_list_t* list)
4053 {
4054 while (list->size() > 0) {
4055 xb_regfree(&list->front());
4056 list->pop_front();
4057 }
4058 }
4059
4060 /************************************************************************
4061 Destroy table filters for partial backup. */
4062 static
4063 void
xb_filters_free()4064 xb_filters_free()
4065 {
4066 xb_regex_list_free(®ex_include_list);
4067 xb_regex_list_free(®ex_exclude_list);
4068
4069 if (tables_include_hash) {
4070 xb_filter_hash_free(tables_include_hash);
4071 }
4072
4073 if (tables_exclude_hash) {
4074 xb_filter_hash_free(tables_exclude_hash);
4075 }
4076
4077 if (databases_include_hash) {
4078 xb_filter_hash_free(databases_include_hash);
4079 }
4080
4081 if (databases_exclude_hash) {
4082 xb_filter_hash_free(databases_exclude_hash);
4083 }
4084 }
4085
4086 /*********************************************************************//**
4087 Create log file metadata. */
4088 static
4089 void
open_or_create_log_file(fil_space_t * space,ulint i)4090 open_or_create_log_file(
4091 /*====================*/
4092 fil_space_t* space,
4093 ulint i) /*!< in: log file number in group */
4094 {
4095 char name[FN_REFLEN];
4096 ulint dirnamelen;
4097
4098 os_normalize_path(srv_log_group_home_dir);
4099
4100 dirnamelen = strlen(srv_log_group_home_dir);
4101 ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile");
4102 memcpy(name, srv_log_group_home_dir, dirnamelen);
4103
4104 /* Add a path separator if needed. */
4105 if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) {
4106 name[dirnamelen++] = OS_PATH_SEPARATOR;
4107 }
4108
4109 sprintf(name + dirnamelen, "%s%zu", "ib_logfile", i);
4110
4111 ut_a(fil_validate());
4112
4113 space->add(name, OS_FILE_CLOSED,
4114 ulint(srv_log_file_size >> srv_page_size_shift),
4115 false, false);
4116 }
4117
4118 /***********************************************************************
4119 Set the open files limit. Based on set_max_open_files().
4120
4121 @return the resulting open files limit. May be less or more than the requested
4122 value. */
4123 static uint
xb_set_max_open_files(uint max_file_limit)4124 xb_set_max_open_files(
4125 /*==================*/
4126 uint max_file_limit) /*!<in: open files limit */
4127 {
4128 #if defined(RLIMIT_NOFILE)
4129 struct rlimit rlimit;
4130 uint old_cur;
4131
4132 if (getrlimit(RLIMIT_NOFILE, &rlimit)) {
4133
4134 goto end;
4135 }
4136
4137 old_cur = (uint) rlimit.rlim_cur;
4138
4139 if (rlimit.rlim_cur == RLIM_INFINITY) {
4140
4141 rlimit.rlim_cur = max_file_limit;
4142 }
4143
4144 if (rlimit.rlim_cur >= max_file_limit) {
4145
4146 max_file_limit = rlimit.rlim_cur;
4147 goto end;
4148 }
4149
4150 rlimit.rlim_cur = rlimit.rlim_max = max_file_limit;
4151
4152 if (setrlimit(RLIMIT_NOFILE, &rlimit)) {
4153
4154 max_file_limit = old_cur; /* Use original value */
4155 } else {
4156
4157 rlimit.rlim_cur = 0; /* Safety if next call fails */
4158
4159 (void) getrlimit(RLIMIT_NOFILE, &rlimit);
4160
4161 if (rlimit.rlim_cur) {
4162
4163 /* If call didn't fail */
4164 max_file_limit = (uint) rlimit.rlim_cur;
4165 }
4166 }
4167
4168 end:
4169 return(max_file_limit);
4170 #else
4171 return(0);
4172 #endif
4173 }
4174
stop_backup_threads()4175 static void stop_backup_threads()
4176 {
4177 if (log_copying_stop && log_copying_running) {
4178 os_event_set(log_copying_stop);
4179 fputs("mariabackup: Stopping log copying thread", stderr);
4180 fflush(stderr);
4181 while (log_copying_running) {
4182 putc('.', stderr);
4183 fflush(stderr);
4184 os_thread_sleep(200000); /*0.2 sec*/
4185 }
4186 putc('\n', stderr);
4187 os_event_destroy(log_copying_stop);
4188 }
4189
4190 if (wait_throttle) {
4191 /* wait for io_watching_thread completion */
4192 while (io_watching_thread_running) {
4193 os_thread_sleep(1000000);
4194 }
4195 os_event_destroy(wait_throttle);
4196 }
4197 }
4198
4199 /** Implement the core of --backup
4200 @return whether the operation succeeded */
xtrabackup_backup_low()4201 static bool xtrabackup_backup_low()
4202 {
4203 ut_ad(!metadata_to_lsn);
4204
4205 /* read the latest checkpoint lsn */
4206 {
4207 ulint max_cp_field;
4208
4209 log_mutex_enter();
4210
4211 if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS
4212 && log_sys.log.format != 0) {
4213 if (max_cp_field == LOG_CHECKPOINT_1) {
4214 log_header_read(max_cp_field);
4215 }
4216 metadata_to_lsn = mach_read_from_8(
4217 log_sys.checkpoint_buf + LOG_CHECKPOINT_LSN);
4218 msg("mariabackup: The latest check point"
4219 " (for incremental): '" LSN_PF "'",
4220 metadata_to_lsn);
4221 } else {
4222 msg("Error: recv_find_max_checkpoint() failed.");
4223 }
4224 log_mutex_exit();
4225 }
4226
4227 stop_backup_threads();
4228
4229 if (metadata_to_lsn && xtrabackup_copy_logfile(true)) {
4230 ds_close(dst_log_file);
4231 dst_log_file = NULL;
4232 return false;
4233 }
4234
4235 if (ds_close(dst_log_file) || !metadata_to_lsn) {
4236 dst_log_file = NULL;
4237 return false;
4238 }
4239
4240 dst_log_file = NULL;
4241
4242 if(!xtrabackup_incremental) {
4243 strcpy(metadata_type, "full-backuped");
4244 metadata_from_lsn = 0;
4245 } else {
4246 strcpy(metadata_type, "incremental");
4247 metadata_from_lsn = incremental_lsn;
4248 }
4249 metadata_last_lsn = log_copy_scanned_lsn;
4250
4251 if (!xtrabackup_stream_metadata(ds_meta)) {
4252 msg("Error: failed to stream metadata.");
4253 return false;
4254 }
4255 if (xtrabackup_extra_lsndir) {
4256 char filename[FN_REFLEN];
4257
4258 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
4259 XTRABACKUP_METADATA_FILENAME);
4260 if (!xtrabackup_write_metadata(filename)) {
4261 msg("Error: failed to write metadata "
4262 "to '%s'.", filename);
4263 return false;
4264 }
4265 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
4266 XTRABACKUP_INFO);
4267 if (!write_xtrabackup_info(mysql_connection, filename, false, false)) {
4268 msg("Error: failed to write info "
4269 "to '%s'.", filename);
4270 return false;
4271 }
4272 }
4273
4274 return true;
4275 }
4276
4277 /** Implement --backup
4278 @return whether the operation succeeded */
xtrabackup_backup_func()4279 static bool xtrabackup_backup_func()
4280 {
4281 MY_STAT stat_info;
4282 uint i;
4283 uint count;
4284 pthread_mutex_t count_mutex;
4285 CorruptedPages corrupted_pages;
4286 data_thread_ctxt_t *data_threads;
4287 pthread_mutex_init(&backup_mutex, NULL);
4288 pthread_cond_init(&scanned_lsn_cond, NULL);
4289
4290 #ifdef USE_POSIX_FADVISE
4291 msg("uses posix_fadvise().");
4292 #endif
4293
4294 /* cd to datadir */
4295
4296 if (my_setwd(mysql_real_data_home,MYF(MY_WME)))
4297 {
4298 msg("my_setwd() failed , %s", mysql_real_data_home);
4299 return(false);
4300 }
4301 msg("cd to %s", mysql_real_data_home);
4302 encryption_plugin_backup_init(mysql_connection);
4303 msg("open files limit requested %u, set to %u",
4304 (uint) xb_open_files_limit,
4305 xb_set_max_open_files(xb_open_files_limit));
4306
4307 mysql_data_home= mysql_data_home_buff;
4308 mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
4309 mysql_data_home[1]=0;
4310
4311 srv_n_purge_threads = 1;
4312 srv_read_only_mode = TRUE;
4313
4314 srv_operation = SRV_OPERATION_BACKUP;
4315 log_file_op = backup_file_op;
4316 metadata_to_lsn = 0;
4317
4318 /* initialize components */
4319 if(innodb_init_param()) {
4320 fail:
4321 metadata_to_lsn = log_copying_running;
4322 stop_backup_threads();
4323 log_file_op = NULL;
4324 if (dst_log_file) {
4325 ds_close(dst_log_file);
4326 dst_log_file = NULL;
4327 }
4328 if (fil_system.is_initialised()) {
4329 innodb_shutdown();
4330 }
4331 return(false);
4332 }
4333
4334 if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
4335 /* Here we still have srv_pool_size counted
4336 in kilobytes (in 4.0 this was in bytes)
4337 srv_boot() converts the value to
4338 pages; if buffer pool is less than 1000 MB,
4339 assume fewer threads. */
4340 srv_max_n_threads = 50000;
4341
4342 } else if (srv_buf_pool_size >= 8 * 1024 * 1024) {
4343
4344 srv_max_n_threads = 10000;
4345 } else {
4346 srv_max_n_threads = 1000; /* saves several MB of memory,
4347 especially in 64-bit
4348 computers */
4349 }
4350
4351 sync_check_init();
4352 ut_d(sync_check_enable());
4353 /* Reset the system variables in the recovery module. */
4354 recv_sys_var_init();
4355 trx_pool_init();
4356
4357 ut_crc32_init();
4358 crc_init();
4359 recv_sys_init();
4360
4361 #ifdef WITH_INNODB_DISALLOW_WRITES
4362 srv_allow_writes_event = os_event_create(0);
4363 os_event_set(srv_allow_writes_event);
4364 #endif
4365
4366 xb_filters_init();
4367
4368 xb_fil_io_init();
4369 srv_n_file_io_threads = srv_n_read_io_threads;
4370
4371 os_aio_init(srv_n_read_io_threads, srv_n_write_io_threads,
4372 SRV_MAX_N_PENDING_SYNC_IOS);
4373
4374 log_sys.create();
4375 log_sys.log.create(srv_n_log_files);
4376 fil_space_t* space = fil_space_create(
4377 "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0,
4378 FIL_TYPE_LOG, NULL);
4379
4380 for (ulint i = 0; i < srv_n_log_files; i++) {
4381 open_or_create_log_file(space, i);
4382 }
4383
4384 /* create extra LSN dir if it does not exist. */
4385 if (xtrabackup_extra_lsndir
4386 &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0))
4387 && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) {
4388 msg("Error: cannot mkdir %d: %s\n",
4389 my_errno, xtrabackup_extra_lsndir);
4390 goto fail;
4391 }
4392
4393 /* create target dir if not exist */
4394 if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
4395 && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){
4396 msg("Error: cannot mkdir %d: %s\n",
4397 my_errno, xtrabackup_target_dir);
4398 goto fail;
4399 }
4400
4401 {
4402 /* definition from recv_recovery_from_checkpoint_start() */
4403 ulint max_cp_field;
4404
4405 /* start back ground thread to copy newer log */
4406 os_thread_id_t log_copying_thread_id;
4407
4408 /* get current checkpoint_lsn */
4409 /* Look for the latest checkpoint from any of the log groups */
4410
4411 log_mutex_enter();
4412
4413 reread_log_header:
4414 dberr_t err = recv_find_max_checkpoint(&max_cp_field);
4415
4416 if (err != DB_SUCCESS) {
4417 msg("Error: cannot read redo log header");
4418 log_mutex_exit();
4419 goto fail;
4420 }
4421
4422 if (log_sys.log.format == 0) {
4423 msg("Error: cannot process redo log before MariaDB 10.2.2");
4424 log_mutex_exit();
4425 goto fail;
4426 }
4427
4428 const byte* buf = log_sys.checkpoint_buf;
4429 checkpoint_lsn_start = log_sys.log.get_lsn();
4430 checkpoint_no_start = log_sys.next_checkpoint_no;
4431
4432 log_header_read(max_cp_field);
4433
4434 if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)
4435 || checkpoint_lsn_start
4436 != mach_read_from_8(buf + LOG_CHECKPOINT_LSN)
4437 || log_sys.log.get_lsn_offset()
4438 != mach_read_from_8(buf + LOG_CHECKPOINT_OFFSET))
4439 goto reread_log_header;
4440
4441 log_mutex_exit();
4442
4443 xtrabackup_init_datasinks();
4444
4445 if (!select_history()) {
4446 goto fail;
4447 }
4448
4449 /* open the log file */
4450 memset(&stat_info, 0, sizeof(MY_STAT));
4451 dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info);
4452 if (dst_log_file == NULL) {
4453 msg("Error: failed to open the target stream for "
4454 "'ib_logfile0'.");
4455 goto fail;
4456 }
4457
4458 /* label it */
4459 byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr_buf[LOG_FILE_HDR_SIZE];
4460 memset(log_hdr_buf, 0, sizeof log_hdr_buf);
4461
4462 byte *log_hdr_field = log_hdr_buf;
4463 mach_write_to_4(LOG_HEADER_FORMAT + log_hdr_field, log_sys.log.format);
4464 mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr_field, log_sys.log.subformat);
4465 mach_write_to_8(LOG_HEADER_START_LSN + log_hdr_field, checkpoint_lsn_start);
4466 strcpy(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr_field),
4467 "Backup " MYSQL_SERVER_VERSION);
4468 log_block_set_checksum(log_hdr_field,
4469 log_block_calc_checksum_crc32(log_hdr_field));
4470
4471 /* copied from log_group_checkpoint() */
4472 log_hdr_field +=
4473 (log_sys.next_checkpoint_no & 1) ? LOG_CHECKPOINT_2 : LOG_CHECKPOINT_1;
4474 /* The least significant bits of LOG_CHECKPOINT_OFFSET must be
4475 stored correctly in the copy of the ib_logfile. The most significant
4476 bits, which identify the start offset of the log block in the file,
4477 we did choose freely, as LOG_FILE_HDR_SIZE. */
4478 ut_ad(!((log_sys.log.get_lsn() ^ checkpoint_lsn_start)
4479 & (OS_FILE_LOG_BLOCK_SIZE - 1)));
4480 /* Adjust the checkpoint page. */
4481 memcpy(log_hdr_field, log_sys.checkpoint_buf, OS_FILE_LOG_BLOCK_SIZE);
4482 mach_write_to_8(log_hdr_field + LOG_CHECKPOINT_OFFSET,
4483 (checkpoint_lsn_start & (OS_FILE_LOG_BLOCK_SIZE - 1))
4484 | LOG_FILE_HDR_SIZE);
4485 log_block_set_checksum(log_hdr_field,
4486 log_block_calc_checksum_crc32(log_hdr_field));
4487
4488 /* Write log header*/
4489 if (ds_write(dst_log_file, log_hdr_buf, sizeof(log_hdr_buf))) {
4490 msg("error: write to logfile failed");
4491 goto fail;
4492 }
4493
4494 log_copying_running = true;
4495 /* start io throttle */
4496 if(xtrabackup_throttle) {
4497 os_thread_id_t io_watching_thread_id;
4498
4499 io_ticket = xtrabackup_throttle;
4500 wait_throttle = os_event_create(0);
4501 io_watching_thread_running = true;
4502
4503 os_thread_create(io_watching_thread, NULL,
4504 &io_watching_thread_id);
4505 }
4506
4507 /* Populate fil_system with tablespaces to copy */
4508 err = xb_load_tablespaces();
4509 if (err != DB_SUCCESS) {
4510 msg("merror: xb_load_tablespaces() failed with"
4511 " error %s.", ut_strerr(err));
4512 fail_before_log_copying_thread_start:
4513 log_copying_running = false;
4514 goto fail;
4515 }
4516
4517 /* copy log file by current position */
4518 log_copy_scanned_lsn = checkpoint_lsn_start;
4519 recv_sys->recovered_lsn = log_copy_scanned_lsn;
4520 log_optimized_ddl_op = backup_optimized_ddl_op;
4521 log_truncate = backup_truncate_fail;
4522
4523 if (xtrabackup_copy_logfile())
4524 goto fail_before_log_copying_thread_start;
4525
4526 DBUG_MARIABACKUP_EVENT("before_innodb_log_copy_thread_started",0);
4527
4528 log_copying_stop = os_event_create(0);
4529 os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
4530
4531 /* FLUSH CHANGED_PAGE_BITMAPS call */
4532 if (!flush_changed_page_bitmaps()) {
4533 goto fail;
4534 }
4535
4536 ut_a(xtrabackup_parallel > 0);
4537
4538 if (xtrabackup_parallel > 1) {
4539 msg("mariabackup: Starting %u threads for parallel data "
4540 "files transfer", xtrabackup_parallel);
4541 }
4542
4543 if (opt_lock_ddl_per_table) {
4544 mdl_lock_all();
4545
4546 DBUG_EXECUTE_IF("check_mdl_lock_works",
4547 dbug_alter_thread_done =
4548 dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
4549 "Waiting for table metadata lock", 1, ER_QUERY_INTERRUPTED););
4550 }
4551
4552 datafiles_iter_t *it = datafiles_iter_new();
4553 if (it == NULL) {
4554 msg("mariabackup: Error: datafiles_iter_new() failed.");
4555 goto fail;
4556 }
4557
4558 /* Create data copying threads */
4559 data_threads = (data_thread_ctxt_t *)
4560 malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel);
4561 count = xtrabackup_parallel;
4562 pthread_mutex_init(&count_mutex, NULL);
4563
4564 for (i = 0; i < (uint) xtrabackup_parallel; i++) {
4565 data_threads[i].it = it;
4566 data_threads[i].num = i+1;
4567 data_threads[i].count = &count;
4568 data_threads[i].count_mutex = &count_mutex;
4569 data_threads[i].corrupted_pages = &corrupted_pages;
4570 os_thread_create(data_copy_thread_func, data_threads + i,
4571 &data_threads[i].id);
4572 }
4573
4574 /* Wait for threads to exit */
4575 while (1) {
4576 os_thread_sleep(1000000);
4577 pthread_mutex_lock(&count_mutex);
4578 bool stop = count == 0;
4579 pthread_mutex_unlock(&count_mutex);
4580 if (stop) {
4581 break;
4582 }
4583 }
4584
4585 pthread_mutex_destroy(&count_mutex);
4586 free(data_threads);
4587 datafiles_iter_free(it);
4588 }
4589
4590 bool ok = backup_start(corrupted_pages);
4591
4592 if (ok) {
4593 ok = xtrabackup_backup_low();
4594
4595 backup_release();
4596
4597 DBUG_EXECUTE_IF("check_mdl_lock_works",
4598 os_event_wait(dbug_alter_thread_done);
4599 os_event_destroy(dbug_alter_thread_done);
4600 );
4601
4602 if (ok) {
4603 backup_finish();
4604 }
4605 }
4606
4607 if (opt_log_innodb_page_corruption)
4608 ok = corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE);
4609
4610 if (!ok) {
4611 goto fail;
4612 }
4613
4614 if (changed_page_bitmap) {
4615 xb_page_bitmap_deinit(changed_page_bitmap);
4616 }
4617 xtrabackup_destroy_datasinks();
4618
4619 msg("Redo log (from LSN " LSN_PF " to " LSN_PF
4620 ") was copied.", checkpoint_lsn_start, log_copy_scanned_lsn);
4621 xb_filters_free();
4622
4623 xb_data_files_close();
4624
4625 /* Make sure that the latest checkpoint was included */
4626 if (metadata_to_lsn > log_copy_scanned_lsn) {
4627 msg("Error: failed to copy enough redo log ("
4628 "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").",
4629 log_copy_scanned_lsn, metadata_to_lsn);
4630 goto fail;
4631 }
4632
4633 innodb_shutdown();
4634 log_file_op = NULL;
4635 pthread_mutex_destroy(&backup_mutex);
4636 pthread_cond_destroy(&scanned_lsn_cond);
4637 if (!corrupted_pages.empty()) {
4638 ut_ad(opt_log_innodb_page_corruption);
4639 msg("Error: corrupted innodb pages are found and logged to "
4640 MB_CORRUPTED_PAGES_FILE " file");
4641 }
4642 return(true);
4643 }
4644
4645
4646 /**
4647 This function handles DDL changes at the end of backup, under protection of
4648 FTWRL. This ensures consistent backup in presence of DDL.
4649
4650 - New tables, that were created during backup, are now copied into backup.
4651 Also, tablespaces with optimized (no redo loggin DDL) are re-copied into
4652 backup. This tablespaces will get the extension ".new" in the backup
4653
4654 - Tables that were renamed during backup, are marked as renamed
4655 For these, file <old_name>.ren will be created.
4656 The content of the file is the new tablespace name.
4657
4658 - Tables that were deleted during backup, are marked as deleted
4659 For these , an empty file <name>.del will be created
4660
4661 It is the responsibility of the prepare phase to deal with .new, .ren, and .del
4662 files.
4663 */
backup_fix_ddl(CorruptedPages & corrupted_pages)4664 void backup_fix_ddl(CorruptedPages &corrupted_pages)
4665 {
4666 std::set<std::string> new_tables;
4667 std::set<std::string> dropped_tables;
4668 std::map<std::string, std::string> renamed_tables;
4669
4670 /* Disable further DDL on backed up tables (only needed for --no-lock).*/
4671 pthread_mutex_lock(&backup_mutex);
4672 log_file_op = backup_file_op_fail;
4673 log_optimized_ddl_op = backup_optimized_ddl_op_fail;
4674 pthread_mutex_unlock(&backup_mutex);
4675
4676 DBUG_MARIABACKUP_EVENT("backup_fix_ddl",0);
4677
4678 for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin();
4679 iter != ddl_tracker.tables_in_backup.end();
4680 iter++) {
4681
4682 const std::string name = iter->second;
4683 ulint id = iter->first;
4684
4685 if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) {
4686 dropped_tables.insert(name);
4687 corrupted_pages.drop_space(id);
4688 continue;
4689 }
4690
4691 bool has_optimized_ddl =
4692 ddl_tracker.optimized_ddl.find(id) != ddl_tracker.optimized_ddl.end();
4693
4694 if (ddl_tracker.id_to_name.find(id) == ddl_tracker.id_to_name.end()) {
4695 if (has_optimized_ddl) {
4696 new_tables.insert(name);
4697 }
4698 continue;
4699 }
4700
4701 /* tablespace was affected by DDL. */
4702 const std::string new_name = ddl_tracker.id_to_name[id];
4703 if (new_name != name) {
4704 if (has_optimized_ddl) {
4705 /* table was renamed, but we need a full copy
4706 of it because of optimized DDL. We emulate a drop/create.*/
4707 dropped_tables.insert(name);
4708 if (opt_log_innodb_page_corruption)
4709 corrupted_pages.drop_space(id);
4710 new_tables.insert(new_name);
4711 } else {
4712 /* Renamed, and no optimized DDL*/
4713 renamed_tables[name] = new_name;
4714 if (opt_log_innodb_page_corruption)
4715 corrupted_pages.rename_space(id, new_name);
4716 }
4717 } else if (has_optimized_ddl) {
4718 /* Table was recreated, or optimized DDL ran.
4719 In both cases we need a full copy in the backup.*/
4720 new_tables.insert(name);
4721 if (opt_log_innodb_page_corruption)
4722 corrupted_pages.drop_space(id);
4723 }
4724 }
4725
4726 /* Find tables that were created during backup (and not removed).*/
4727 for(space_id_to_name_t::iterator iter = ddl_tracker.id_to_name.begin();
4728 iter != ddl_tracker.id_to_name.end();
4729 iter++) {
4730
4731 ulint id = iter->first;
4732 std::string name = iter->second;
4733
4734 if (ddl_tracker.tables_in_backup.find(id) != ddl_tracker.tables_in_backup.end()) {
4735 /* already processed above */
4736 continue;
4737 }
4738
4739 if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end()) {
4740 dropped_tables.erase(name);
4741 new_tables.insert(name);
4742 if (opt_log_innodb_page_corruption)
4743 corrupted_pages.drop_space(id);
4744 }
4745 }
4746
4747 // Mark tablespaces for rename
4748 for (std::map<std::string, std::string>::iterator iter = renamed_tables.begin();
4749 iter != renamed_tables.end(); ++iter) {
4750 const std::string old_name = iter->first;
4751 std::string new_name = iter->second;
4752 backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str());
4753 }
4754
4755 // Mark tablespaces for drop
4756 for (std::set<std::string>::iterator iter = dropped_tables.begin();
4757 iter != dropped_tables.end();
4758 iter++) {
4759 const std::string name(*iter);
4760 backup_file_printf((name + ".del").c_str(), "%s", "");
4761 }
4762
4763 // Load and copy new tables.
4764 // Close all datanodes first, reload only new tables.
4765 std::vector<fil_node_t *> all_nodes;
4766 datafiles_iter_t *it = datafiles_iter_new();
4767 if (!it)
4768 return;
4769 while (fil_node_t *node = datafiles_iter_next(it)) {
4770 all_nodes.push_back(node);
4771 }
4772 for (size_t i = 0; i < all_nodes.size(); i++) {
4773 fil_node_t *n = all_nodes[i];
4774 if (n->space->id == 0)
4775 continue;
4776 if (n->is_open()) {
4777 mutex_enter(&fil_system.mutex);
4778 n->close();
4779 mutex_exit(&fil_system.mutex);
4780 }
4781 fil_space_free(n->space->id, false);
4782 }
4783 datafiles_iter_free(it);
4784
4785 for (std::set<std::string>::iterator iter = new_tables.begin();
4786 iter != new_tables.end(); iter++) {
4787 const char *space_name = iter->c_str();
4788 if (check_if_skip_table(space_name))
4789 continue;
4790 xb_load_single_table_tablespace(*iter, false);
4791 }
4792
4793 it = datafiles_iter_new();
4794 if (!it)
4795 return;
4796
4797 while (fil_node_t *node = datafiles_iter_next(it)) {
4798 fil_space_t * space = node->space;
4799 if (!fil_is_user_tablespace_id(space->id))
4800 continue;
4801 std::string dest_name(node->space->name);
4802 dest_name.append(".new");
4803 xtrabackup_copy_datafile(node, 0, dest_name.c_str(), wf_write_through,
4804 corrupted_pages);
4805 }
4806
4807 datafiles_iter_free(it);
4808 }
4809
4810 /* ================= prepare ================= */
4811
4812 /***********************************************************************
4813 Generates path to the meta file path from a given path to an incremental .delta
4814 by replacing trailing ".delta" with ".meta", or returns error if 'delta_path'
4815 does not end with the ".delta" character sequence.
4816 @return TRUE on success, FALSE on error. */
4817 static
4818 ibool
get_meta_path(const char * delta_path,char * meta_path)4819 get_meta_path(
4820 const char *delta_path, /* in: path to a .delta file */
4821 char *meta_path) /* out: path to the corresponding .meta
4822 file */
4823 {
4824 size_t len = strlen(delta_path);
4825
4826 if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) {
4827 return FALSE;
4828 }
4829 memcpy(meta_path, delta_path, len - 6);
4830 strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX);
4831
4832 return TRUE;
4833 }
4834
4835 /****************************************************************//**
4836 Create a new tablespace on disk and return the handle to its opened
4837 file. Code adopted from fil_create_new_single_table_tablespace with
4838 the main difference that only disk file is created without updating
4839 the InnoDB in-memory dictionary data structures.
4840
4841 @return true on success, false on error. */
4842 static
4843 bool
xb_space_create_file(const char * path,ulint space_id,ulint flags,pfs_os_file_t * file)4844 xb_space_create_file(
4845 /*==================*/
4846 const char* path, /*!<in: path to tablespace */
4847 ulint space_id, /*!<in: space id */
4848 ulint flags, /*!<in: tablespace flags */
4849 pfs_os_file_t* file) /*!<out: file handle */
4850 {
4851 bool ret;
4852 byte* buf;
4853 byte* page;
4854
4855 *file = os_file_create_simple_no_error_handling(
4856 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret);
4857 if (!ret) {
4858 msg("Can't create file %s", path);
4859 return ret;
4860 }
4861
4862 ret = os_file_set_size(path, *file,
4863 FIL_IBD_FILE_INITIAL_SIZE
4864 << srv_page_size_shift);
4865 if (!ret) {
4866 msg("mariabackup: cannot set size for file %s", path);
4867 os_file_close(*file);
4868 os_file_delete(0, path);
4869 return ret;
4870 }
4871
4872 buf = static_cast<byte *>(malloc(3U << srv_page_size_shift));
4873 /* Align the memory for file i/o if we might have O_DIRECT set */
4874 page = static_cast<byte *>(ut_align(buf, srv_page_size));
4875
4876 memset(page, '\0', srv_page_size);
4877
4878 fsp_header_init_fields(page, space_id, flags);
4879 mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id);
4880
4881 const page_size_t page_size(flags);
4882
4883 if (!page_size.is_compressed()) {
4884 buf_flush_init_for_writing(NULL, page, NULL, 0);
4885
4886 ret = os_file_write(IORequestWrite, path, *file, page, 0,
4887 srv_page_size);
4888 } else {
4889 page_zip_des_t page_zip;
4890 ulint zip_size = page_size.physical();
4891 page_zip_set_size(&page_zip, zip_size);
4892 page_zip.data = page + srv_page_size;
4893 fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size);
4894
4895 #ifdef UNIV_DEBUG
4896 page_zip.m_start =
4897 #endif /* UNIV_DEBUG */
4898 page_zip.m_end = page_zip.m_nonempty =
4899 page_zip.n_blobs = 0;
4900
4901 buf_flush_init_for_writing(NULL, page, &page_zip, 0);
4902
4903 ret = os_file_write(IORequestWrite, path, *file,
4904 page_zip.data, 0, zip_size);
4905 }
4906
4907 free(buf);
4908
4909 if (ret != DB_SUCCESS) {
4910 msg("mariabackup: could not write the first page to %s",
4911 path);
4912 os_file_close(*file);
4913 os_file_delete(0, path);
4914 return ret;
4915 }
4916
4917 return TRUE;
4918 }
4919
fil_space_get_by_name(const char * name)4920 static fil_space_t* fil_space_get_by_name(const char* name)
4921 {
4922 ut_ad(mutex_own(&fil_system.mutex));
4923 for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system.space_list);
4924 space != NULL;
4925 space = UT_LIST_GET_NEXT(space_list, space))
4926 if (!strcmp(space->name, name)) return space;
4927 return NULL;
4928 }
4929
4930 /***********************************************************************
4931 Searches for matching tablespace file for given .delta file and space_id
4932 in given directory. When matching tablespace found, renames it to match the
4933 name of .delta file. If there was a tablespace with matching name and
4934 mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no
4935 matching file, creates a new tablespace.
4936 @return file handle of matched or created file */
4937 static
4938 pfs_os_file_t
xb_delta_open_matching_space(const char * dbname,const char * name,const xb_delta_info_t & info,char * real_name,size_t real_name_len,bool * success)4939 xb_delta_open_matching_space(
4940 const char* dbname, /* in: path to destination database dir */
4941 const char* name, /* in: name of delta file (without .delta) */
4942 const xb_delta_info_t& info,
4943 char* real_name, /* out: full path of destination file */
4944 size_t real_name_len, /* out: buffer size for real_name */
4945 bool* success) /* out: indicates error. true = success */
4946 {
4947 char dest_dir[FN_REFLEN];
4948 char dest_space_name[FN_REFLEN];
4949 fil_space_t* fil_space;
4950 pfs_os_file_t file;
4951 xb_filter_entry_t* table;
4952
4953 ut_a(dbname != NULL ||
4954 !fil_is_user_tablespace_id(info.space_id) ||
4955 info.space_id == ULINT_UNDEFINED);
4956
4957 *success = false;
4958
4959 if (dbname) {
4960 snprintf(dest_dir, FN_REFLEN, "%s/%s",
4961 xtrabackup_target_dir, dbname);
4962 os_normalize_path(dest_dir);
4963
4964 snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name);
4965 } else {
4966 snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir);
4967 os_normalize_path(dest_dir);
4968
4969 snprintf(dest_space_name, FN_REFLEN, "%s", name);
4970 }
4971
4972 snprintf(real_name, real_name_len,
4973 "%s/%s",
4974 xtrabackup_target_dir, dest_space_name);
4975 os_normalize_path(real_name);
4976 /* Truncate ".ibd" */
4977 dest_space_name[strlen(dest_space_name) - 4] = '\0';
4978
4979 /* Create the database directory if it doesn't exist yet */
4980 if (!os_file_create_directory(dest_dir, FALSE)) {
4981 msg("mariabackup: error: cannot create dir %s", dest_dir);
4982 return file;
4983 }
4984
4985 log_mutex_enter();
4986 if (!fil_is_user_tablespace_id(info.space_id)) {
4987 found:
4988 /* open the file and return its handle */
4989
4990 file = os_file_create_simple_no_error_handling(
4991 0, real_name,
4992 OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success);
4993
4994 if (!*success) {
4995 msg("mariabackup: Cannot open file %s\n", real_name);
4996 }
4997 exit:
4998 log_mutex_exit();
4999 return file;
5000 }
5001
5002 /* remember space name for further reference */
5003 table = static_cast<xb_filter_entry_t *>
5004 (malloc(sizeof(xb_filter_entry_t) +
5005 strlen(dest_space_name) + 1));
5006
5007 table->name = ((char*)table) + sizeof(xb_filter_entry_t);
5008 strcpy(table->name, dest_space_name);
5009 HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash,
5010 ut_fold_string(table->name), table);
5011
5012 mutex_enter(&fil_system.mutex);
5013 fil_space = fil_space_get_by_name(dest_space_name);
5014 mutex_exit(&fil_system.mutex);
5015
5016 if (fil_space != NULL) {
5017 if (fil_space->id == info.space_id
5018 || info.space_id == ULINT_UNDEFINED) {
5019 /* we found matching space */
5020 goto found;
5021 } else {
5022
5023 char tmpname[FN_REFLEN];
5024
5025 snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#" ULINTPF,
5026 dbname, fil_space->id);
5027
5028 msg("mariabackup: Renaming %s to %s.ibd",
5029 fil_space->name, tmpname);
5030
5031 if (fil_space->rename(tmpname, NULL, false)
5032 != DB_SUCCESS) {
5033 msg("mariabackup: Cannot rename %s to %s",
5034 fil_space->name, tmpname);
5035 goto exit;
5036 }
5037 }
5038 }
5039
5040 if (info.space_id == ULINT_UNDEFINED)
5041 {
5042 die("Can't handle DDL operation on tablespace "
5043 "%s\n", dest_space_name);
5044 }
5045 mutex_enter(&fil_system.mutex);
5046 fil_space = fil_space_get_by_id(info.space_id);
5047 mutex_exit(&fil_system.mutex);
5048 if (fil_space != NULL) {
5049 char tmpname[FN_REFLEN];
5050
5051 strncpy(tmpname, dest_space_name, FN_REFLEN);
5052
5053 msg("mariabackup: Renaming %s to %s",
5054 fil_space->name, dest_space_name);
5055
5056 if (fil_space->rename(tmpname, NULL, false) != DB_SUCCESS)
5057 {
5058 msg("mariabackup: Cannot rename %s to %s",
5059 fil_space->name, dest_space_name);
5060 goto exit;
5061 }
5062
5063 goto found;
5064 }
5065
5066 /* No matching space found. create the new one. */
5067 const ulint flags = info.page_size.is_compressed()
5068 ? get_bit_shift(info.page_size.physical()
5069 >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
5070 << FSP_FLAGS_POS_ZIP_SSIZE
5071 | FSP_FLAGS_MASK_POST_ANTELOPE
5072 | FSP_FLAGS_MASK_ATOMIC_BLOBS
5073 | (info.page_size.logical() == UNIV_PAGE_SIZE_ORIG
5074 ? 0
5075 : get_bit_shift(info.page_size.logical()
5076 >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
5077 << FSP_FLAGS_POS_PAGE_SSIZE)
5078 : FSP_FLAGS_PAGE_SSIZE();
5079 ut_ad(page_size_t(flags).equals_to(info.page_size));
5080
5081 if (fil_space_create(dest_space_name, info.space_id, flags,
5082 FIL_TYPE_TABLESPACE, 0)) {
5083 *success = xb_space_create_file(real_name, info.space_id,
5084 flags, &file);
5085 } else {
5086 msg("Can't create tablespace %s\n", dest_space_name);
5087 }
5088
5089 goto exit;
5090 }
5091
5092 /************************************************************************
5093 Applies a given .delta file to the corresponding data file.
5094 @return TRUE on success */
5095 static
5096 ibool
xtrabackup_apply_delta(const char * dirname,const char * dbname,const char * filename,void *)5097 xtrabackup_apply_delta(
5098 const char* dirname, /* in: dir name of incremental */
5099 const char* dbname, /* in: database name (ibdata: NULL) */
5100 const char* filename, /* in: file name (not a path),
5101 including the .delta extension */
5102 void* /*data*/)
5103 {
5104 pfs_os_file_t src_file;
5105 pfs_os_file_t dst_file;
5106 char src_path[FN_REFLEN];
5107 char dst_path[FN_REFLEN];
5108 char meta_path[FN_REFLEN];
5109 char space_name[FN_REFLEN];
5110 bool success;
5111
5112 ibool last_buffer = FALSE;
5113 ulint page_in_buffer;
5114 ulint incremental_buffers = 0;
5115
5116 xb_delta_info_t info(univ_page_size, SRV_TMP_SPACE_ID);
5117 ulint page_size;
5118 ulint page_size_shift;
5119 byte* incremental_buffer_base = NULL;
5120 byte* incremental_buffer;
5121
5122 size_t offset;
5123
5124 ut_a(xtrabackup_incremental);
5125
5126 if (dbname) {
5127 snprintf(src_path, sizeof(src_path), "%s/%s/%s",
5128 dirname, dbname, filename);
5129 snprintf(dst_path, sizeof(dst_path), "%s/%s/%s",
5130 xtrabackup_real_target_dir, dbname, filename);
5131 } else {
5132 snprintf(src_path, sizeof(src_path), "%s/%s",
5133 dirname, filename);
5134 snprintf(dst_path, sizeof(dst_path), "%s/%s",
5135 xtrabackup_real_target_dir, filename);
5136 }
5137 dst_path[strlen(dst_path) - 6] = '\0';
5138
5139 strncpy(space_name, filename, FN_REFLEN - 1);
5140 space_name[FN_REFLEN - 1] = '\0';
5141 space_name[strlen(space_name) - 6] = 0;
5142
5143 if (!get_meta_path(src_path, meta_path)) {
5144 goto error;
5145 }
5146
5147 os_normalize_path(dst_path);
5148 os_normalize_path(src_path);
5149 os_normalize_path(meta_path);
5150
5151 if (!xb_read_delta_metadata(meta_path, &info)) {
5152 goto error;
5153 }
5154
5155 page_size = info.page_size.physical();
5156 page_size_shift = get_bit_shift(page_size);
5157 msg("page size for %s is %zu bytes",
5158 src_path, page_size);
5159 if (page_size_shift < 10 ||
5160 page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) {
5161 msg("error: invalid value of page_size "
5162 "(%zu bytes) read from %s", page_size, meta_path);
5163 goto error;
5164 }
5165
5166 src_file = os_file_create_simple_no_error_handling(
5167 0, src_path,
5168 OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success);
5169 if (!success) {
5170 os_file_get_last_error(TRUE);
5171 msg("error: can't open %s", src_path);
5172 goto error;
5173 }
5174
5175 posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
5176
5177 dst_file = xb_delta_open_matching_space(
5178 dbname, space_name, info,
5179 dst_path, sizeof(dst_path), &success);
5180 if (!success) {
5181 msg("error: can't open %s", dst_path);
5182 goto error;
5183 }
5184
5185 posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
5186
5187 /* allocate buffer for incremental backup (4096 pages) */
5188 incremental_buffer_base = static_cast<byte *>
5189 (malloc((page_size / 4 + 1) * page_size));
5190 incremental_buffer = static_cast<byte *>
5191 (ut_align(incremental_buffer_base,
5192 page_size));
5193
5194 msg("Applying %s to %s...", src_path, dst_path);
5195
5196 while (!last_buffer) {
5197 ulint cluster_header;
5198
5199 /* read to buffer */
5200 /* first block of block cluster */
5201 offset = ((incremental_buffers * (page_size / 4))
5202 << page_size_shift);
5203 success = os_file_read(IORequestRead, src_file,
5204 incremental_buffer, offset, page_size);
5205 if (success != DB_SUCCESS) {
5206 goto error;
5207 }
5208
5209 cluster_header = mach_read_from_4(incremental_buffer);
5210 switch(cluster_header) {
5211 case 0x78747261UL: /*"xtra"*/
5212 break;
5213 case 0x58545241UL: /*"XTRA"*/
5214 last_buffer = TRUE;
5215 break;
5216 default:
5217 msg("error: %s seems not "
5218 ".delta file.", src_path);
5219 goto error;
5220 }
5221
5222 /* FIXME: If the .delta modifies FSP_SIZE on page 0,
5223 extend the file to that size. */
5224
5225 for (page_in_buffer = 1; page_in_buffer < page_size / 4;
5226 page_in_buffer++) {
5227 if (mach_read_from_4(incremental_buffer + page_in_buffer * 4)
5228 == 0xFFFFFFFFUL)
5229 break;
5230 }
5231
5232 ut_a(last_buffer || page_in_buffer == page_size / 4);
5233
5234 /* read whole of the cluster */
5235 success = os_file_read(IORequestRead, src_file,
5236 incremental_buffer,
5237 offset, page_in_buffer * page_size);
5238 if (success != DB_SUCCESS) {
5239 goto error;
5240 }
5241
5242 posix_fadvise(src_file, offset, page_in_buffer * page_size,
5243 POSIX_FADV_DONTNEED);
5244
5245 for (page_in_buffer = 1; page_in_buffer < page_size / 4;
5246 page_in_buffer++) {
5247 ulint offset_on_page;
5248
5249 offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4);
5250
5251 if (offset_on_page == 0xFFFFFFFFUL)
5252 break;
5253
5254 uchar *buf = incremental_buffer + page_in_buffer * page_size;
5255 const os_offset_t off = os_offset_t(offset_on_page)*page_size;
5256
5257 if (off == 0) {
5258 /* Read tablespace size from page 0,
5259 and extend the file to specified size.*/
5260 os_offset_t n_pages = mach_read_from_4(
5261 buf + FSP_HEADER_OFFSET + FSP_SIZE);
5262 if (mach_read_from_4(buf
5263 + FIL_PAGE_SPACE_ID)) {
5264 if (!os_file_set_size(
5265 dst_path, dst_file,
5266 n_pages * page_size))
5267 goto error;
5268 } else if (fil_space_t* space
5269 = fil_system.sys_space) {
5270 /* The system tablespace can
5271 consist of multiple files. The
5272 first one has full tablespace
5273 size in page 0, but only the last
5274 file should be extended. */
5275 fil_node_t* n = UT_LIST_GET_FIRST(
5276 space->chain);
5277 bool fail = !strcmp(n->name, dst_path)
5278 && !fil_space_extend(
5279 space, (ulint)n_pages);
5280 if (fail) goto error;
5281 }
5282 }
5283
5284 success = os_file_write(IORequestWrite,
5285 dst_path, dst_file, buf, off, page_size);
5286 if (success != DB_SUCCESS) {
5287 goto error;
5288 }
5289 }
5290
5291 /* Free file system buffer cache after the batch was written. */
5292 #ifdef __linux__
5293 os_file_flush_func(dst_file);
5294 #endif
5295 posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
5296
5297
5298 incremental_buffers++;
5299 }
5300
5301 free(incremental_buffer_base);
5302 if (src_file != OS_FILE_CLOSED) {
5303 os_file_close(src_file);
5304 os_file_delete(0,src_path);
5305 }
5306 if (dst_file != OS_FILE_CLOSED)
5307 os_file_close(dst_file);
5308 return TRUE;
5309
5310 error:
5311 free(incremental_buffer_base);
5312 if (src_file != OS_FILE_CLOSED)
5313 os_file_close(src_file);
5314 if (dst_file != OS_FILE_CLOSED)
5315 os_file_close(dst_file);
5316 msg("Error: xtrabackup_apply_delta(): "
5317 "failed to apply %s to %s.\n", src_path, dst_path);
5318 return FALSE;
5319 }
5320
5321
change_extension(std::string filename,std::string new_ext)5322 std::string change_extension(std::string filename, std::string new_ext) {
5323 DBUG_ASSERT(new_ext.size() == 3);
5324 std::string new_name(filename);
5325 new_name.resize(new_name.size() - new_ext.size());
5326 new_name.append(new_ext);
5327 return new_name;
5328 }
5329
5330
rename_file(const char * from,const char * to)5331 static void rename_file(const char *from,const char *to) {
5332 msg("Renaming %s to %s\n", from, to);
5333 if (my_rename(from, to, MY_WME)) {
5334 die("Can't rename %s to %s errno %d", from, to, errno);
5335 }
5336 }
5337
rename_file(const std::string & from,const std::string & to)5338 static void rename_file(const std::string& from, const std::string &to) {
5339 rename_file(from.c_str(), to.c_str());
5340 }
5341 /************************************************************************
5342 Callback to handle datadir entry. Function of this type will be called
5343 for each entry which matches the mask by xb_process_datadir.
5344 @return should return TRUE on success */
5345 typedef ibool (*handle_datadir_entry_func_t)(
5346 /*=========================================*/
5347 const char* data_home_dir, /*!<in: path to datadir */
5348 const char* db_name, /*!<in: database name */
5349 const char* file_name, /*!<in: file name with suffix */
5350 void* arg); /*!<in: caller-provided data */
5351
5352 /** Rename, and replace destination file, if exists */
rename_force(const char * from,const char * to)5353 static void rename_force(const char *from, const char *to) {
5354 if (access(to, R_OK) == 0) {
5355 msg("Removing %s", to);
5356 if (my_delete(to, MYF(MY_WME))) {
5357 msg("Can't remove %s, errno %d", to, errno);
5358 exit(EXIT_FAILURE);
5359 }
5360 }
5361 rename_file(from,to);
5362 }
5363
5364
5365 /** During prepare phase, rename ".new" files, that were created in
5366 backup_fix_ddl() and backup_optimized_ddl_op(), to ".ibd". In the case of
5367 incremental backup, i.e. of arg argument is set, move ".new" files to
5368 destination directory and rename them to ".ibd", remove existing ".ibd.delta"
5369 and ".idb.meta" files in incremental directory to avoid applying delta to
5370 ".ibd" file.
5371
5372 @param[in] data_home_dir path to datadir
5373 @param[in] db_name database name
5374 @param[in] file_name file name with suffix
5375 @param[in] arg destination path, used in incremental backup to notify, that
5376 *.new file must be moved to destibation directory
5377
5378 @return true */
prepare_handle_new_files(const char * data_home_dir,const char * db_name,const char * file_name,void * arg)5379 static ibool prepare_handle_new_files(const char *data_home_dir,
5380 const char *db_name,
5381 const char *file_name, void *arg)
5382 {
5383 const char *dest_dir = static_cast<const char *>(arg);
5384 std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/' + file_name;
5385 /* Copy "*.new" files from incremental to base dir for incremental backup */
5386 std::string dest_path=
5387 dest_dir ? std::string(dest_dir) + '/' + std::string(db_name) +
5388 '/' + file_name : src_path;
5389
5390 size_t index = dest_path.find(".new");
5391 DBUG_ASSERT(index != std::string::npos);
5392 dest_path.replace(index, strlen(".ibd"), ".ibd");
5393 rename_force(src_path.c_str(),dest_path.c_str());
5394
5395 if (dest_dir) {
5396 /* remove delta and meta files to avoid delta applying for new file */
5397 index = src_path.find(".new");
5398 DBUG_ASSERT(index != std::string::npos);
5399 src_path.replace(index, std::string::npos, ".ibd.delta");
5400 if (access(src_path.c_str(), R_OK) == 0) {
5401 msg("Removing %s", src_path.c_str());
5402 if (my_delete(src_path.c_str(), MYF(MY_WME)))
5403 die("Can't remove %s, errno %d", src_path.c_str(), errno);
5404 }
5405 src_path.replace(index, std::string::npos, ".ibd.meta");
5406 if (access(src_path.c_str(), R_OK) == 0) {
5407 msg("Removing %s", src_path.c_str());
5408 if (my_delete(src_path.c_str(), MYF(MY_WME)))
5409 die("Can't remove %s, errno %d", src_path.c_str(), errno);
5410 }
5411
5412 /* add table name to the container to avoid it's deletion at the end of
5413 prepare */
5414 std::string table_name = std::string(db_name) + '/'
5415 + std::string(file_name, file_name + strlen(file_name) - strlen(".new"));
5416 xb_filter_entry_t *table = static_cast<xb_filter_entry_t *>
5417 (malloc(sizeof(xb_filter_entry_t) + table_name.size() + 1));
5418 table->name = ((char*)table) + sizeof(xb_filter_entry_t);
5419 strcpy(table->name, table_name.c_str());
5420 HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash,
5421 ut_fold_string(table->name), table);
5422 }
5423
5424 return TRUE;
5425 }
5426
5427 /************************************************************************
5428 Callback to handle datadir entry. Deletes entry if it has no matching
5429 fil_space in fil_system directory.
5430 @return FALSE if delete attempt was unsuccessful */
5431 static
5432 ibool
rm_if_not_found(const char * data_home_dir,const char * db_name,const char * file_name,void * arg)5433 rm_if_not_found(
5434 const char* data_home_dir, /*!<in: path to datadir */
5435 const char* db_name, /*!<in: database name */
5436 const char* file_name, /*!<in: file name with suffix */
5437 void* arg __attribute__((unused)))
5438 {
5439 char name[FN_REFLEN];
5440 xb_filter_entry_t* table;
5441
5442 snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name);
5443 /* Truncate ".ibd" */
5444 name[strlen(name) - 4] = '\0';
5445
5446 HASH_SEARCH(name_hash, inc_dir_tables_hash, ut_fold_string(name),
5447 xb_filter_entry_t*,
5448 table, (void) 0,
5449 !strcmp(table->name, name));
5450
5451 if (!table) {
5452 snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir,
5453 db_name, file_name);
5454 return os_file_delete(0, name);
5455 }
5456
5457 return(TRUE);
5458 }
5459
5460 /** Function enumerates files in datadir (provided by path) which are matched
5461 by provided suffix. For each entry callback is called.
5462
5463 @param[in] path datadir path
5464 @param[in] suffix suffix to match against
5465 @param[in] func callback
5466 @param[in] func_arg arguments for the above callback
5467
5468 @return FALSE if callback for some entry returned FALSE */
xb_process_datadir(const char * path,const char * suffix,handle_datadir_entry_func_t func,void * func_arg=NULL)5469 static ibool xb_process_datadir(const char *path, const char *suffix,
5470 handle_datadir_entry_func_t func,
5471 void *func_arg = NULL)
5472 {
5473 ulint ret;
5474 char dbpath[OS_FILE_MAX_PATH+1];
5475 os_file_dir_t dir;
5476 os_file_dir_t dbdir;
5477 os_file_stat_t dbinfo;
5478 os_file_stat_t fileinfo;
5479 ulint suffix_len;
5480 dberr_t err = DB_SUCCESS;
5481 static char current_dir[2];
5482
5483 current_dir[0] = FN_CURLIB;
5484 current_dir[1] = 0;
5485 srv_data_home = current_dir;
5486
5487 suffix_len = strlen(suffix);
5488
5489 /* datafile */
5490 dbdir = os_file_opendir(path, FALSE);
5491
5492 if (dbdir != NULL) {
5493 ret = fil_file_readdir_next_file(&err, path, dbdir,
5494 &fileinfo);
5495 while (ret == 0) {
5496 if (fileinfo.type == OS_FILE_TYPE_DIR) {
5497 goto next_file_item_1;
5498 }
5499
5500 if (strlen(fileinfo.name) > suffix_len
5501 && 0 == strcmp(fileinfo.name +
5502 strlen(fileinfo.name) - suffix_len,
5503 suffix)) {
5504 if (!func(
5505 path, NULL,
5506 fileinfo.name, func_arg))
5507 {
5508 os_file_closedir(dbdir);
5509 return(FALSE);
5510 }
5511 }
5512 next_file_item_1:
5513 ret = fil_file_readdir_next_file(&err,
5514 path, dbdir,
5515 &fileinfo);
5516 }
5517
5518 os_file_closedir(dbdir);
5519 } else {
5520 msg("Can't open dir %s", path);
5521 }
5522
5523 /* single table tablespaces */
5524 dir = os_file_opendir(path, FALSE);
5525
5526 if (dir == NULL) {
5527 msg("Can't open dir %s", path);
5528 }
5529
5530 ret = fil_file_readdir_next_file(&err, path, dir,
5531 &dbinfo);
5532 while (ret == 0) {
5533 if (dbinfo.type == OS_FILE_TYPE_FILE
5534 || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
5535
5536 goto next_datadir_item;
5537 }
5538
5539 snprintf(dbpath, sizeof(dbpath), "%.*s/%.*s",
5540 OS_FILE_MAX_PATH/2-1,
5541 path,
5542 OS_FILE_MAX_PATH/2-1,
5543 dbinfo.name);
5544
5545 os_normalize_path(dbpath);
5546
5547 dbdir = os_file_opendir(dbpath, FALSE);
5548
5549 if (dbdir != NULL) {
5550
5551 ret = fil_file_readdir_next_file(&err, dbpath, dbdir,
5552 &fileinfo);
5553 while (ret == 0) {
5554
5555 if (fileinfo.type == OS_FILE_TYPE_DIR) {
5556
5557 goto next_file_item_2;
5558 }
5559
5560 if (strlen(fileinfo.name) > suffix_len
5561 && 0 == strcmp(fileinfo.name +
5562 strlen(fileinfo.name) -
5563 suffix_len,
5564 suffix)) {
5565 /* The name ends in suffix; process
5566 the file */
5567 if (!func(
5568 path,
5569 dbinfo.name,
5570 fileinfo.name, func_arg))
5571 {
5572 os_file_closedir(dbdir);
5573 os_file_closedir(dir);
5574 return(FALSE);
5575 }
5576 }
5577 next_file_item_2:
5578 ret = fil_file_readdir_next_file(&err,
5579 dbpath, dbdir,
5580 &fileinfo);
5581 }
5582
5583 os_file_closedir(dbdir);
5584 }
5585 next_datadir_item:
5586 ret = fil_file_readdir_next_file(&err,
5587 path,
5588 dir, &dbinfo);
5589 }
5590
5591 os_file_closedir(dir);
5592
5593 return(TRUE);
5594 }
5595
5596 /************************************************************************
5597 Applies all .delta files from incremental_dir to the full backup.
5598 @return TRUE on success. */
5599 static
5600 ibool
xtrabackup_apply_deltas()5601 xtrabackup_apply_deltas()
5602 {
5603 return xb_process_datadir(xtrabackup_incremental_dir, ".delta",
5604 xtrabackup_apply_delta);
5605 }
5606
5607
5608 static
5609 void
innodb_free_param()5610 innodb_free_param()
5611 {
5612 srv_sys_space.shutdown();
5613 free_tmpdir(&mysql_tmpdir_list);
5614 }
5615
5616
5617 /** Check if file exists*/
file_exists(std::string name)5618 static bool file_exists(std::string name)
5619 {
5620 return access(name.c_str(), R_OK) == 0 ;
5621 }
5622
5623 /** Read file content into STL string */
read_file_as_string(const std::string file)5624 static std::string read_file_as_string(const std::string file) {
5625 char content[FN_REFLEN];
5626 FILE *f = fopen(file.c_str(), "r");
5627 if (!f) {
5628 msg("Can not open %s", file.c_str());
5629 }
5630 size_t len = fread(content, 1, FN_REFLEN, f);
5631 fclose(f);
5632 return std::string(content, len);
5633 }
5634
5635 /** Delete file- Provide verbose diagnostics and exit, if operation fails. */
delete_file(const std::string & file,bool if_exists=false)5636 static void delete_file(const std::string& file, bool if_exists = false) {
5637 if (if_exists && !file_exists(file))
5638 return;
5639 if (my_delete(file.c_str(), MYF(MY_WME))) {
5640 die("Can't remove %s, errno %d", file.c_str(), errno);
5641 }
5642 }
5643
5644 /**
5645 Rename tablespace during prepare.
5646 Backup in its end phase may generate some .ren files, recording
5647 tablespaces that should be renamed in --prepare.
5648 */
rename_table_in_prepare(const std::string & datadir,const std::string & from,const std::string & to,const char * extension=0)5649 static void rename_table_in_prepare(const std::string &datadir, const std::string& from , const std::string& to,
5650 const char *extension=0) {
5651 if (!extension) {
5652 static const char *extensions_nonincremental[] = { ".ibd", 0 };
5653 static const char *extensions_incremental[] = { ".ibd.delta", ".ibd.meta", 0 };
5654 const char **extensions = xtrabackup_incremental_dir ?
5655 extensions_incremental : extensions_nonincremental;
5656 for (size_t i = 0; extensions[i]; i++) {
5657 rename_table_in_prepare(datadir, from, to, extensions[i]);
5658 }
5659 return;
5660 }
5661 std::string src = std::string(datadir) + "/" + from + extension;
5662 std::string dest = std::string(datadir) + "/" + to + extension;
5663 std::string ren2, tmp;
5664 if (file_exists(dest)) {
5665 ren2= std::string(datadir) + "/" + to + ".ren";
5666 if (!file_exists(ren2)) {
5667 msg("ERROR : File %s was not found, but expected during rename processing\n", ren2.c_str());
5668 ut_a(0);
5669 }
5670 tmp = to + "#";
5671 rename_table_in_prepare(datadir, to, tmp);
5672 }
5673 rename_file(src, dest);
5674 if (ren2.size()) {
5675 // Make sure the temp. renamed file is processed.
5676 std::string to2 = read_file_as_string(ren2);
5677 rename_table_in_prepare(datadir, tmp, to2);
5678 delete_file(ren2);
5679 }
5680 }
5681
prepare_handle_ren_files(const char * datadir,const char * db,const char * filename,void *)5682 static ibool prepare_handle_ren_files(const char *datadir, const char *db, const char *filename, void *) {
5683
5684 std::string ren_file = std::string(datadir) + "/" + db + "/" + filename;
5685 if (!file_exists(ren_file))
5686 return TRUE;
5687
5688 std::string to = read_file_as_string(ren_file);
5689 std::string source_space_name = std::string(db) + "/" + filename;
5690 source_space_name.resize(source_space_name.size() - 4); // remove extension
5691
5692 rename_table_in_prepare(datadir, source_space_name.c_str(), to.c_str());
5693 delete_file(ren_file);
5694 return TRUE;
5695 }
5696
5697 /* Remove tablespaces during backup, based on */
prepare_handle_del_files(const char * datadir,const char * db,const char * filename,void *)5698 static ibool prepare_handle_del_files(const char *datadir, const char *db, const char *filename, void *) {
5699 std::string del_file = std::string(datadir) + "/" + db + "/" + filename;
5700 std::string path(del_file);
5701 path.resize(path.size() - 4); // remove extension;
5702 if (xtrabackup_incremental) {
5703 delete_file(path + ".ibd.delta", true);
5704 delete_file(path + ".ibd.meta", true);
5705 }
5706 else {
5707 delete_file(path + ".ibd", true);
5708 }
5709 delete_file(del_file);
5710 return TRUE;
5711 }
5712
5713 /** Implement --prepare
5714 @return whether the operation succeeded */
xtrabackup_prepare_func(char ** argv)5715 static bool xtrabackup_prepare_func(char** argv)
5716 {
5717 CorruptedPages corrupted_pages;
5718 char metadata_path[FN_REFLEN];
5719
5720 /* cd to target-dir */
5721
5722 if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME)))
5723 {
5724 msg("can't my_setwd %s", xtrabackup_real_target_dir);
5725 return(false);
5726 }
5727 msg("cd to %s", xtrabackup_real_target_dir);
5728
5729 fil_path_to_mysql_datadir = ".";
5730
5731 ut_ad(xtrabackup_incremental == xtrabackup_incremental_dir);
5732 if (xtrabackup_incremental) {
5733 inc_dir_tables_hash = hash_create(1000);
5734 ut_ad(inc_dir_tables_hash);
5735 }
5736
5737 msg("open files limit requested %u, set to %u",
5738 (uint) xb_open_files_limit,
5739 xb_set_max_open_files(xb_open_files_limit));
5740
5741 /* Fix DDL for prepare. Process .del,.ren, and .new files.
5742 The order in which files are processed, is important
5743 (see MDEV-18185, MDEV-18201)
5744 */
5745 xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".",
5746 ".del", prepare_handle_del_files);
5747 xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".",
5748 ".ren", prepare_handle_ren_files);
5749 if (xtrabackup_incremental_dir) {
5750 xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files);
5751 xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files);
5752 xb_process_datadir(xtrabackup_incremental_dir, ".new",
5753 prepare_handle_new_files, (void *)".");
5754 }
5755 else {
5756 xb_process_datadir(".", ".new", prepare_handle_new_files);
5757 }
5758
5759 int argc; for (argc = 0; argv[argc]; argc++) {}
5760 encryption_plugin_prepare_init(argc, argv);
5761
5762 xtrabackup_target_dir= mysql_data_home_buff;
5763 xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here
5764 xtrabackup_target_dir[1]=0;
5765 const lsn_t target_lsn = xtrabackup_incremental
5766 ? incremental_to_lsn : metadata_to_lsn;
5767
5768 /*
5769 read metadata of target
5770 */
5771 sprintf(metadata_path, "%s/%s", xtrabackup_target_dir,
5772 XTRABACKUP_METADATA_FILENAME);
5773
5774 if (!xtrabackup_read_metadata(metadata_path)) {
5775 msg("Error: failed to read metadata from '%s'\n",
5776 metadata_path);
5777 return(false);
5778 }
5779
5780 if (!strcmp(metadata_type, "full-backuped")) {
5781 if (xtrabackup_incremental) {
5782 msg("error: applying incremental backup "
5783 "needs a prepared target.");
5784 return(false);
5785 }
5786 msg("This target seems to be not prepared yet.");
5787 } else if (!strcmp(metadata_type, "log-applied")) {
5788 msg("This target seems to be already prepared.");
5789 } else {
5790 msg("This target does not have correct metadata.");
5791 return(false);
5792 }
5793
5794 bool ok = !xtrabackup_incremental
5795 || metadata_to_lsn == incremental_lsn;
5796 if (!ok) {
5797 msg("error: This incremental backup seems "
5798 "not to be proper for the target. Check 'to_lsn' of the target and "
5799 "'from_lsn' of the incremental.");
5800 return(false);
5801 }
5802
5803 srv_max_n_threads = 1000;
5804 srv_undo_logs = 1;
5805 srv_n_purge_threads = 1;
5806
5807 xb_filters_init();
5808
5809 srv_log_group_home_dir = NULL;
5810 srv_thread_concurrency = 1;
5811
5812 if (xtrabackup_incremental) {
5813 srv_operation = SRV_OPERATION_RESTORE_DELTA;
5814
5815 if (innodb_init_param()) {
5816 goto error_cleanup;
5817 }
5818
5819 sync_check_init();
5820 ut_d(sync_check_enable());
5821 ut_crc32_init();
5822 recv_sys_init();
5823 log_sys.create();
5824 recv_recovery_on = true;
5825
5826 #ifdef WITH_INNODB_DISALLOW_WRITES
5827 srv_allow_writes_event = os_event_create(0);
5828 os_event_set(srv_allow_writes_event);
5829 #endif
5830 dberr_t err = xb_data_files_init();
5831 if (err != DB_SUCCESS) {
5832 msg("mariabackup: error: xb_data_files_init() failed "
5833 "with error %s\n", ut_strerr(err));
5834 goto error_cleanup;
5835 }
5836
5837 ok = xtrabackup_apply_deltas();
5838
5839 xb_data_files_close();
5840
5841 if (ok) {
5842 /* Cleanup datadir from tablespaces deleted
5843 between full and incremental backups */
5844
5845 xb_process_datadir("./", ".ibd", rm_if_not_found);
5846 }
5847
5848 xb_filter_hash_free(inc_dir_tables_hash);
5849
5850 fil_system.close();
5851 #ifdef WITH_INNODB_DISALLOW_WRITES
5852 os_event_destroy(srv_allow_writes_event);
5853 #endif
5854 innodb_free_param();
5855 log_sys.close();
5856 sync_check_close();
5857 if (!ok) goto error_cleanup;
5858 }
5859
5860 srv_operation=
5861 xtrabackup_export
5862 ? SRV_OPERATION_RESTORE_EXPORT
5863 : (xtrabackup_rollback_xa ? SRV_OPERATION_RESTORE_ROLLBACK_XA
5864 : SRV_OPERATION_RESTORE);
5865
5866 if (innodb_init_param()) {
5867 goto error_cleanup;
5868 }
5869
5870 /* increase IO threads */
5871 if (srv_n_file_io_threads < 10) {
5872 srv_n_read_io_threads = 4;
5873 srv_n_write_io_threads = 4;
5874 }
5875
5876 msg("Starting InnoDB instance for recovery.");
5877
5878 msg("mariabackup: Using %lld bytes for buffer pool "
5879 "(set by --use-memory parameter)", xtrabackup_use_memory);
5880
5881 srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct;
5882
5883 if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) {
5884 srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct;
5885 }
5886
5887 if (xtrabackup_rollback_xa)
5888 srv_fast_shutdown= 0;
5889
5890 if (innodb_init()) {
5891 goto error_cleanup;
5892 }
5893
5894 corrupted_pages.read_from_file(MB_CORRUPTED_PAGES_FILE);
5895 if (xtrabackup_incremental)
5896 {
5897 char inc_filename[FN_REFLEN];
5898 sprintf(inc_filename, "%s/%s", xtrabackup_incremental_dir,
5899 MB_CORRUPTED_PAGES_FILE);
5900 corrupted_pages.read_from_file(inc_filename);
5901 }
5902 if (!corrupted_pages.empty())
5903 corrupted_pages.zero_out_free_pages();
5904 if (corrupted_pages.empty())
5905 {
5906 if (!xtrabackup_incremental && unlink(MB_CORRUPTED_PAGES_FILE) &&
5907 errno != ENOENT)
5908 {
5909 char errbuf[MYSYS_STRERROR_SIZE];
5910 my_strerror(errbuf, sizeof(errbuf), errno);
5911 die("Error: unlink %s failed: %s", MB_CORRUPTED_PAGES_FILE,
5912 errbuf);
5913 }
5914 }
5915 else
5916 corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE);
5917
5918 if (xtrabackup_rollback_xa)
5919 {
5920 /* Please do not merge MDEV-21168 fix in 10.5+ */
5921 compile_time_assert(MYSQL_VERSION_ID < 10 * 10000 + 5 * 100);
5922 XID *xid_list=
5923 (XID *) my_malloc(MAX_XID_LIST_SIZE * sizeof(XID), MYF(0));
5924 if (!xid_list)
5925 {
5926 msg("Can't allocate %i bytes for XID's list", MAX_XID_LIST_SIZE);
5927 ok= false;
5928 goto error_cleanup;
5929 }
5930 int got;
5931 ut_ad(recv_no_log_write);
5932 ut_d(recv_no_log_write= false);
5933 while ((got= trx_recover_for_mysql(xid_list, MAX_XID_LIST_SIZE)) > 0)
5934 {
5935 for (int i= 0; i < got; i++)
5936 {
5937 #ifndef DBUG_OFF
5938 int rc=
5939 #endif // !DBUG_OFF
5940 innobase_rollback_by_xid(NULL, xid_list + i);
5941 #ifndef DBUG_OFF
5942 if (rc == 0)
5943 {
5944 char buf[XIDDATASIZE * 4 + 6]; // see xid_to_str
5945 DBUG_PRINT("info",
5946 ("rollback xid %s", xid_to_str(buf, xid_list[i])));
5947 }
5948 #endif // !DBUG_OFF
5949 }
5950 }
5951 ut_d(recv_no_log_write= true);
5952 my_free(xid_list);
5953 }
5954
5955 if (ok) {
5956 msg("Last binlog file %s, position %lld",
5957 trx_sys.recovered_binlog_filename,
5958 longlong(trx_sys.recovered_binlog_offset));
5959 }
5960
5961 /* Check whether the log is applied enough or not. */
5962 if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID))
5963 && srv_start_lsn < target_lsn) {
5964 msg("mariabackup: error: "
5965 "The log was only applied up to LSN " LSN_PF
5966 ", instead of " LSN_PF,
5967 srv_start_lsn, target_lsn);
5968 ok = false;
5969 }
5970 #ifdef WITH_WSREP
5971 else if (ok) xb_write_galera_info(xtrabackup_incremental);
5972 #endif
5973
5974 if (xtrabackup_rollback_xa)
5975 {
5976 // See innobase_end() and thd_destructor_proxy()
5977 while (srv_fast_shutdown == 0 &&
5978 (trx_sys.any_active_transactions() ||
5979 static_cast<uint>(thread_count) > srv_n_purge_threads + 1))
5980 os_thread_sleep(1000);
5981
5982 srv_shutdown_bg_undo_sources();
5983 srv_purge_shutdown();
5984 buf_flush_sync_all_buf_pools();
5985 }
5986
5987 innodb_shutdown();
5988 innodb_free_param();
5989
5990 /* output to metadata file */
5991 if (ok) {
5992 char filename[FN_REFLEN];
5993
5994 strcpy(metadata_type, "log-applied");
5995
5996 if(xtrabackup_incremental
5997 && metadata_to_lsn < incremental_to_lsn)
5998 {
5999 metadata_to_lsn = incremental_to_lsn;
6000 metadata_last_lsn = incremental_last_lsn;
6001 }
6002
6003 sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
6004 if (!xtrabackup_write_metadata(filename)) {
6005
6006 msg("mariabackup: Error: failed to write metadata "
6007 "to '%s'", filename);
6008 ok = false;
6009 } else if (xtrabackup_extra_lsndir) {
6010 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
6011 if (!xtrabackup_write_metadata(filename)) {
6012 msg("mariabackup: Error: failed to write "
6013 "metadata to '%s'", filename);
6014 ok = false;
6015 }
6016 }
6017 }
6018
6019 if (ok) ok = apply_log_finish();
6020
6021 if (ok && xtrabackup_export)
6022 ok= (prepare_export() == 0);
6023
6024 error_cleanup:
6025 xb_filters_free();
6026 return ok && !ib::error::was_logged() && corrupted_pages.empty();
6027 }
6028
6029 /**************************************************************************
6030 Append group name to xb_load_default_groups list. */
6031 static
6032 void
append_defaults_group(const char * group,const char * default_groups[],size_t default_groups_size)6033 append_defaults_group(const char *group, const char *default_groups[],
6034 size_t default_groups_size)
6035 {
6036 uint i;
6037 bool appended = false;
6038 for (i = 0; i < default_groups_size - 1; i++) {
6039 if (default_groups[i] == NULL) {
6040 default_groups[i] = group;
6041 appended = true;
6042 break;
6043 }
6044 }
6045 ut_a(appended);
6046 }
6047
6048 static const char*
normalize_privilege_target_name(const char * name)6049 normalize_privilege_target_name(const char* name)
6050 {
6051 if (strcmp(name, "*") == 0) {
6052 return "\\*";
6053 }
6054 else {
6055 /* should have no regex special characters. */
6056 ut_ad(strpbrk(name, ".()[]*+?") == 0);
6057 }
6058 return name;
6059 }
6060
6061 /******************************************************************//**
6062 Check if specific privilege is granted.
6063 Uses regexp magic to check if requested privilege is granted for given
6064 database.table or database.* or *.*
6065 or if user has 'ALL PRIVILEGES' granted.
6066 @return true if requested privilege is granted, false otherwise. */
6067 static bool
has_privilege(const std::list<std::string> & granted,const char * required,const char * db_name,const char * table_name)6068 has_privilege(const std::list<std::string> &granted,
6069 const char* required,
6070 const char* db_name,
6071 const char* table_name)
6072 {
6073 char buffer[1000];
6074 regex_t priv_re;
6075 regmatch_t tables_regmatch[1];
6076 bool result = false;
6077
6078 db_name = normalize_privilege_target_name(db_name);
6079 table_name = normalize_privilege_target_name(table_name);
6080
6081 int written = snprintf(buffer, sizeof(buffer),
6082 "GRANT .*(%s)|(ALL PRIVILEGES).* ON (\\*|`%s`)\\.(\\*|`%s`)",
6083 required, db_name, table_name);
6084 if (written < 0 || written == sizeof(buffer)
6085 || regcomp(&priv_re, buffer, REG_EXTENDED)) {
6086 die("regcomp() failed for '%s'", buffer);
6087 }
6088
6089 typedef std::list<std::string>::const_iterator string_iter;
6090 for (string_iter i = granted.begin(), e = granted.end(); i != e; ++i) {
6091 int res = regexec(&priv_re, i->c_str(),
6092 1, tables_regmatch, 0);
6093
6094 if (res != REG_NOMATCH) {
6095 result = true;
6096 break;
6097 }
6098 }
6099
6100 xb_regfree(&priv_re);
6101 return result;
6102 }
6103
6104 enum {
6105 PRIVILEGE_OK = 0,
6106 PRIVILEGE_WARNING = 1,
6107 PRIVILEGE_ERROR = 2,
6108 };
6109
6110 /******************************************************************//**
6111 Check if specific privilege is granted.
6112 Prints error message if required privilege is missing.
6113 @return PRIVILEGE_OK if requested privilege is granted, error otherwise. */
6114 static
check_privilege(const std::list<std::string> & granted_priv,const char * required,const char * target_database,const char * target_table,int error=PRIVILEGE_ERROR)6115 int check_privilege(
6116 const std::list<std::string> &granted_priv, /* in: list of
6117 granted privileges*/
6118 const char* required, /* in: required privilege name */
6119 const char* target_database, /* in: required privilege target
6120 database name */
6121 const char* target_table, /* in: required privilege target
6122 table name */
6123 int error = PRIVILEGE_ERROR) /* in: return value if privilege
6124 is not granted */
6125 {
6126 if (!has_privilege(granted_priv,
6127 required, target_database, target_table)) {
6128 msg("%s: missing required privilege %s on %s.%s",
6129 (error == PRIVILEGE_ERROR ? "Error" : "Warning"),
6130 required, target_database, target_table);
6131 return error;
6132 }
6133 return PRIVILEGE_OK;
6134 }
6135
6136
6137 /******************************************************************//**
6138 Check DB user privileges according to the intended actions.
6139
6140 Fetches DB user privileges, determines intended actions based on
6141 command-line arguments and prints missing privileges.
6142 May terminate application with EXIT_FAILURE exit code.*/
6143 static void
check_all_privileges()6144 check_all_privileges()
6145 {
6146 if (!mysql_connection) {
6147 /* Not connected, no queries is going to be executed. */
6148 return;
6149 }
6150
6151 /* Fetch effective privileges. */
6152 std::list<std::string> granted_privileges;
6153 MYSQL_ROW row = 0;
6154 MYSQL_RES* result = xb_mysql_query(mysql_connection, "SHOW GRANTS",
6155 true);
6156 while ((row = mysql_fetch_row(result))) {
6157 granted_privileges.push_back(*row);
6158 }
6159 mysql_free_result(result);
6160
6161 int check_result = PRIVILEGE_OK;
6162
6163 /* FLUSH TABLES WITH READ LOCK */
6164 if (!opt_no_lock)
6165 {
6166 check_result |= check_privilege(
6167 granted_privileges,
6168 "RELOAD", "*", "*");
6169 }
6170
6171 if (!opt_no_lock)
6172 {
6173 check_result |= check_privilege(
6174 granted_privileges,
6175 "PROCESS", "*", "*");
6176 }
6177
6178 /* KILL ... */
6179 if ((!opt_no_lock && (opt_kill_long_queries_timeout || opt_lock_ddl_per_table))
6180 /* START SLAVE SQL_THREAD */
6181 /* STOP SLAVE SQL_THREAD */
6182 || opt_safe_slave_backup) {
6183 check_result |= check_privilege(
6184 granted_privileges,
6185 "SUPER", "*", "*",
6186 PRIVILEGE_WARNING);
6187 }
6188
6189 /* SHOW MASTER STATUS */
6190 /* SHOW SLAVE STATUS */
6191 if (opt_galera_info || opt_slave_info
6192 || (opt_no_lock && opt_safe_slave_backup)) {
6193 check_result |= check_privilege(granted_privileges,
6194 "REPLICATION CLIENT", "*", "*",
6195 PRIVILEGE_WARNING);
6196 }
6197
6198 if (check_result & PRIVILEGE_ERROR) {
6199 mysql_close(mysql_connection);
6200 msg("Current privileges, as reported by 'SHOW GRANTS': ");
6201 int n=1;
6202 for (std::list<std::string>::const_iterator it = granted_privileges.begin();
6203 it != granted_privileges.end();
6204 it++,n++) {
6205 msg(" %d.%s", n, it->c_str());
6206 }
6207 die("Insufficient privileges");
6208 }
6209 }
6210
6211 bool
xb_init()6212 xb_init()
6213 {
6214 const char *mixed_options[4] = {NULL, NULL, NULL, NULL};
6215 int n_mixed_options;
6216
6217 /* sanity checks */
6218
6219 if (opt_slave_info
6220 && opt_no_lock
6221 && !opt_safe_slave_backup) {
6222 msg("Error: --slave-info is used with --no-lock but "
6223 "without --safe-slave-backup. The binlog position "
6224 "cannot be consistent with the backup data.");
6225 return(false);
6226 }
6227
6228 if (xtrabackup_backup && opt_rsync)
6229 {
6230 if (xtrabackup_stream_fmt)
6231 {
6232 msg("Error: --rsync doesn't work with --stream\n");
6233 return(false);
6234 }
6235 bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0));
6236 if (!have_rsync)
6237 {
6238 msg("Error: rsync executable not found, cannot run backup with --rsync\n");
6239 return false;
6240 }
6241 }
6242
6243 n_mixed_options = 0;
6244
6245 if (opt_decompress) {
6246 mixed_options[n_mixed_options++] = "--decompress";
6247 }
6248
6249 if (xtrabackup_copy_back) {
6250 mixed_options[n_mixed_options++] = "--copy-back";
6251 }
6252
6253 if (xtrabackup_move_back) {
6254 mixed_options[n_mixed_options++] = "--move-back";
6255 }
6256
6257 if (xtrabackup_prepare) {
6258 mixed_options[n_mixed_options++] = "--apply-log";
6259 }
6260
6261 if (n_mixed_options > 1) {
6262 msg("Error: %s and %s are mutually exclusive\n",
6263 mixed_options[0], mixed_options[1]);
6264 return(false);
6265 }
6266
6267 if (xtrabackup_backup) {
6268 if ((mysql_connection = xb_mysql_connect()) == NULL) {
6269 return(false);
6270 }
6271
6272 if (!get_mysql_vars(mysql_connection)) {
6273 return(false);
6274 }
6275 if (opt_check_privileges) {
6276 check_all_privileges();
6277 }
6278 history_start_time = time(NULL);
6279
6280 }
6281
6282 return(true);
6283 }
6284
6285
6286 extern void init_signals(void);
6287
6288 #include <sql_locale.h>
6289
6290
setup_error_messages()6291 void setup_error_messages()
6292 {
6293 my_default_lc_messages = &my_locale_en_US;
6294 if (init_errmessage())
6295 die("could not initialize error messages");
6296 }
6297
6298 void
handle_options(int argc,char ** argv,char *** argv_client,char *** argv_server)6299 handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
6300 {
6301 /* Setup some variables for Innodb.*/
6302
6303 srv_operation = SRV_OPERATION_RESTORE;
6304
6305 files_charset_info = &my_charset_utf8_general_ci;
6306
6307
6308 setup_error_messages();
6309 sys_var_init();
6310 plugin_mutex_init();
6311 mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
6312 opt_stack_trace = 1;
6313 test_flags |= TEST_SIGINT;
6314 init_signals();
6315 #ifndef _WIN32
6316 /* Exit process on SIGINT. */
6317 my_sigset(SIGINT, SIG_DFL);
6318 #endif
6319
6320 sf_leaking_memory = 1; /* don't report memory leaks on early exist */
6321
6322 int i;
6323 int ho_error;
6324
6325 char* target_dir = NULL;
6326 bool prepare = false;
6327
6328 char conf_file[FN_REFLEN];
6329 int argc_client = argc;
6330 int argc_server = argc;
6331
6332 /* scan options for group and config file to load defaults from */
6333 for (i = 1; i < argc; i++) {
6334
6335 char *optend = strcend(argv[i], '=');
6336
6337 if (strncmp(argv[i], "--defaults-group",
6338 optend - argv[i]) == 0) {
6339 defaults_group = optend + 1;
6340 append_defaults_group(defaults_group,
6341 xb_server_default_groups,
6342 array_elements(xb_server_default_groups));
6343 }
6344
6345 if (strncmp(argv[i], "--login-path",
6346 optend - argv[i]) == 0) {
6347 append_defaults_group(optend + 1,
6348 xb_client_default_groups,
6349 array_elements(xb_client_default_groups));
6350 }
6351
6352 if (!strncmp(argv[i], "--prepare",
6353 optend - argv[i])) {
6354 prepare = true;
6355 }
6356
6357 if (!strncmp(argv[i], "--apply-log",
6358 optend - argv[i])) {
6359 prepare = true;
6360 }
6361
6362 if (!strncmp(argv[i], "--target-dir",
6363 optend - argv[i]) && *optend) {
6364 target_dir = optend + 1;
6365 }
6366
6367 if (!*optend && argv[i][0] != '-') {
6368 target_dir = argv[i];
6369 }
6370 }
6371
6372 snprintf(conf_file, sizeof(conf_file), "my");
6373
6374 if (prepare && target_dir) {
6375 snprintf(conf_file, sizeof(conf_file),
6376 "%s/backup-my.cnf", target_dir);
6377 if (!strncmp(argv[1], "--defaults-file=", 16)) {
6378 /* Remove defaults-file*/
6379 for (int i = 2; ; i++) {
6380 if ((argv[i-1]= argv[i]) == 0)
6381 break;
6382 }
6383 argc--;
6384 }
6385 }
6386
6387 *argv_client = argv;
6388 *argv_server = argv;
6389 load_defaults_or_exit(conf_file, xb_server_default_groups,
6390 &argc_server, argv_server);
6391
6392 int n;
6393 for (n = 0; (*argv_server)[n]; n++) {};
6394 argc_server = n;
6395
6396 print_param_str <<
6397 "# This MySQL options file was generated by XtraBackup.\n"
6398 "[" << defaults_group << "]\n";
6399
6400 /* We want xtrabackup to ignore unknown options, because it only
6401 recognizes a small subset of server variables */
6402 my_getopt_skip_unknown = TRUE;
6403
6404 /* Reset u_max_value for all options, as we don't want the
6405 --maximum-... modifier to set the actual option values */
6406 for (my_option *optp= xb_server_options; optp->name; optp++) {
6407 optp->u_max_value = (G_PTR *) &global_max_value;
6408 }
6409
6410
6411 /* Throw a descriptive error if --defaults-file or --defaults-extra-file
6412 is not the first command line argument */
6413 for (int i = 2 ; i < argc ; i++) {
6414 char *optend = strcend((argv)[i], '=');
6415
6416 if (optend - argv[i] == 15 &&
6417 !strncmp(argv[i], "--defaults-file", optend - argv[i])) {
6418 die("--defaults-file must be specified first on the command line");
6419 }
6420 if (optend - argv[i] == 21 &&
6421 !strncmp(argv[i], "--defaults-extra-file",
6422 optend - argv[i])) {
6423 die("--defaults-extra-file must be specified first on the command line");
6424 }
6425 }
6426
6427 if (argc_server > 0
6428 && (ho_error=handle_options(&argc_server, argv_server,
6429 xb_server_options, xb_get_one_option)))
6430 exit(ho_error);
6431
6432 load_defaults_or_exit(conf_file, xb_client_default_groups,
6433 &argc_client, argv_client);
6434
6435 for (n = 0; (*argv_client)[n]; n++) {};
6436 argc_client = n;
6437
6438 if (innobackupex_mode && argc_client > 0) {
6439 /* emulate innobackupex script */
6440 innobackupex_mode = true;
6441 if (!ibx_handle_options(&argc_client, argv_client)) {
6442 exit(EXIT_FAILURE);
6443 }
6444 }
6445
6446 if (argc_client > 0
6447 && (ho_error=handle_options(&argc_client, argv_client,
6448 xb_client_options, xb_get_one_option)))
6449 exit(ho_error);
6450
6451 /* Reject command line arguments that don't look like options, i.e. are
6452 not of the form '-X' (single-character options) or '--option' (long
6453 options) */
6454 for (int i = 0 ; i < argc_client ; i++) {
6455 const char * const opt = (*argv_client)[i];
6456
6457 if (strncmp(opt, "--", 2) &&
6458 !(strlen(opt) == 2 && opt[0] == '-')) {
6459 bool server_option = true;
6460
6461 for (int j = 0; j < argc_server; j++) {
6462 if (opt == (*argv_server)[j]) {
6463 server_option = false;
6464 break;
6465 }
6466 }
6467
6468 if (!server_option) {
6469 msg("mariabackup: Error:"
6470 " unknown argument: '%s'", opt);
6471 exit(EXIT_FAILURE);
6472 }
6473 }
6474 }
6475 }
6476
6477 static int main_low(char** argv);
6478 static int get_exepath(char *buf, size_t size, const char *argv0);
6479
6480 /* ================= main =================== */
main(int argc,char ** argv)6481 int main(int argc, char **argv)
6482 {
6483 char **client_defaults, **server_defaults;
6484
6485 my_getopt_prefix_matching= 0;
6486
6487 if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0]))
6488 strncpy(mariabackup_exe,argv[0], FN_REFLEN-1);
6489
6490
6491 if (argc > 1 )
6492 {
6493 /* In "prepare export", we need to start mysqld
6494 Since it is not always be installed on the machine,
6495 we start "mariabackup --mysqld", which acts as mysqld
6496 */
6497 if (strcmp(argv[1], "--mysqld") == 0)
6498 {
6499 extern int mysqld_main(int argc, char **argv);
6500 argc--;
6501 argv++;
6502 argv[0]+=2;
6503 return mysqld_main(argc, argv);
6504 }
6505 if(strcmp(argv[1], "--innobackupex") == 0)
6506 {
6507 argv++;
6508 argc--;
6509 innobackupex_mode = true;
6510 }
6511 }
6512
6513 if (argc > 1)
6514 strncpy(orig_argv1,argv[1],sizeof(orig_argv1) -1);
6515
6516 init_signals();
6517 MY_INIT(argv[0]);
6518
6519 pthread_key_create(&THR_THD, NULL);
6520 my_pthread_setspecific_ptr(THR_THD, NULL);
6521
6522 xb_regex_init();
6523
6524 capture_tool_command(argc, argv);
6525
6526 if (mysql_server_init(-1, NULL, NULL))
6527 {
6528 die("mysql_server_init() failed");
6529 }
6530
6531 system_charset_info = &my_charset_utf8_general_ci;
6532 key_map_full.set_all();
6533
6534 logger.init_base();
6535 logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE);
6536 mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log,
6537 MY_MUTEX_INIT_FAST);
6538
6539 handle_options(argc, argv, &client_defaults, &server_defaults);
6540
6541 #ifndef DBUG_OFF
6542 if (dbug_option) {
6543 DBUG_SET_INITIAL(dbug_option);
6544 DBUG_SET(dbug_option);
6545 }
6546 #endif
6547
6548 int status = main_low(server_defaults);
6549
6550 backup_cleanup();
6551
6552 if (innobackupex_mode) {
6553 ibx_cleanup();
6554 }
6555
6556 free_defaults(client_defaults);
6557 free_defaults(server_defaults);
6558
6559 #ifndef DBUG_OFF
6560 if (dbug_option) {
6561 DBUG_END();
6562 }
6563 #endif
6564
6565 if (THR_THD)
6566 (void) pthread_key_delete(THR_THD);
6567
6568 logger.cleanup_base();
6569 cleanup_errmsgs();
6570 free_error_messages();
6571 mysql_mutex_destroy(&LOCK_error_log);
6572
6573 if (status == EXIT_SUCCESS) {
6574 msg("completed OK!");
6575 }
6576
6577 return status;
6578 }
6579
main_low(char ** argv)6580 static int main_low(char** argv)
6581 {
6582 if (innobackupex_mode) {
6583 if (!ibx_init()) {
6584 return(EXIT_FAILURE);
6585 }
6586 }
6587
6588 if (!xtrabackup_print_param && !xtrabackup_prepare
6589 && !strcmp(mysql_data_home, "./")) {
6590 if (!xtrabackup_print_param)
6591 usage();
6592 msg("mariabackup: Error: Please set parameter 'datadir'");
6593 return(EXIT_FAILURE);
6594 }
6595
6596 /* Expand target-dir, incremental-basedir, etc. */
6597
6598 char cwd[FN_REFLEN];
6599 my_getwd(cwd, sizeof(cwd), MYF(0));
6600
6601 my_load_path(xtrabackup_real_target_dir,
6602 xtrabackup_target_dir, cwd);
6603 unpack_dirname(xtrabackup_real_target_dir,
6604 xtrabackup_real_target_dir);
6605 xtrabackup_target_dir= xtrabackup_real_target_dir;
6606
6607 if (xtrabackup_incremental_basedir) {
6608 my_load_path(xtrabackup_real_incremental_basedir,
6609 xtrabackup_incremental_basedir, cwd);
6610 unpack_dirname(xtrabackup_real_incremental_basedir,
6611 xtrabackup_real_incremental_basedir);
6612 xtrabackup_incremental_basedir =
6613 xtrabackup_real_incremental_basedir;
6614 }
6615
6616 if (xtrabackup_incremental_dir) {
6617 my_load_path(xtrabackup_real_incremental_dir,
6618 xtrabackup_incremental_dir, cwd);
6619 unpack_dirname(xtrabackup_real_incremental_dir,
6620 xtrabackup_real_incremental_dir);
6621 xtrabackup_incremental_dir = xtrabackup_real_incremental_dir;
6622 }
6623
6624 if (xtrabackup_extra_lsndir) {
6625 my_load_path(xtrabackup_real_extra_lsndir,
6626 xtrabackup_extra_lsndir, cwd);
6627 unpack_dirname(xtrabackup_real_extra_lsndir,
6628 xtrabackup_real_extra_lsndir);
6629 xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir;
6630 }
6631
6632 /* get default temporary directory */
6633 if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
6634 opt_mysql_tmpdir = getenv("TMPDIR");
6635 #if defined(__WIN__)
6636 if (!opt_mysql_tmpdir) {
6637 opt_mysql_tmpdir = getenv("TEMP");
6638 }
6639 if (!opt_mysql_tmpdir) {
6640 opt_mysql_tmpdir = getenv("TMP");
6641 }
6642 #endif
6643 if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
6644 opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR);
6645 }
6646 }
6647
6648 /* temporary setting of enough size */
6649 srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX;
6650 srv_page_size = UNIV_PAGE_SIZE_MAX;
6651 if (xtrabackup_backup && xtrabackup_incremental) {
6652 /* direct specification is only for --backup */
6653 /* and the lsn is prior to the other option */
6654
6655 char* endchar;
6656 int error = 0;
6657 incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10);
6658 if (*endchar != '\0')
6659 error = 1;
6660
6661 if (error) {
6662 msg("mariabackup: value '%s' may be wrong format for "
6663 "incremental option.", xtrabackup_incremental);
6664 return(EXIT_FAILURE);
6665 }
6666 } else if (xtrabackup_backup && xtrabackup_incremental_basedir) {
6667 char filename[FN_REFLEN];
6668
6669 sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME);
6670
6671 if (!xtrabackup_read_metadata(filename)) {
6672 msg("mariabackup: error: failed to read metadata from "
6673 "%s", filename);
6674 return(EXIT_FAILURE);
6675 }
6676
6677 incremental_lsn = metadata_to_lsn;
6678 xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy
6679 } else if (xtrabackup_prepare && xtrabackup_incremental_dir) {
6680 char filename[FN_REFLEN];
6681
6682 sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME);
6683
6684 if (!xtrabackup_read_metadata(filename)) {
6685 msg("mariabackup: error: failed to read metadata from "
6686 "%s", filename);
6687 return(EXIT_FAILURE);
6688 }
6689
6690 incremental_lsn = metadata_from_lsn;
6691 incremental_to_lsn = metadata_to_lsn;
6692 incremental_last_lsn = metadata_last_lsn;
6693 xtrabackup_incremental = xtrabackup_incremental_dir; //dummy
6694
6695 } else if (opt_incremental_history_name) {
6696 xtrabackup_incremental = opt_incremental_history_name;
6697 } else if (opt_incremental_history_uuid) {
6698 xtrabackup_incremental = opt_incremental_history_uuid;
6699 } else {
6700 xtrabackup_incremental = NULL;
6701 }
6702
6703 if (xtrabackup_stream && !xtrabackup_backup) {
6704 msg("Warning: --stream parameter is ignored, it only works together with --backup.");
6705 }
6706
6707 if (!xb_init()) {
6708 return(EXIT_FAILURE);
6709 }
6710
6711 /* --print-param */
6712 if (xtrabackup_print_param) {
6713 printf("%s", print_param_str.str().c_str());
6714 return(EXIT_SUCCESS);
6715 }
6716
6717 print_version();
6718 if (xtrabackup_incremental) {
6719 msg("incremental backup from " LSN_PF " is enabled.",
6720 incremental_lsn);
6721 }
6722
6723 if (xtrabackup_export && innobase_file_per_table == FALSE) {
6724 msg("mariabackup: auto-enabling --innodb-file-per-table due to "
6725 "the --export option");
6726 innobase_file_per_table = TRUE;
6727 }
6728
6729 /* cannot execute both for now */
6730 {
6731 int num = 0;
6732
6733 if (xtrabackup_backup) num++;
6734 if (xtrabackup_prepare) num++;
6735 if (xtrabackup_copy_back) num++;
6736 if (xtrabackup_move_back) num++;
6737 if (xtrabackup_decrypt_decompress) num++;
6738 if (num != 1) { /* !XOR (for now) */
6739 usage();
6740 return(EXIT_FAILURE);
6741 }
6742 }
6743
6744 /* --backup */
6745 if (xtrabackup_backup && !xtrabackup_backup_func()) {
6746 return(EXIT_FAILURE);
6747 }
6748
6749 /* --prepare */
6750 if (xtrabackup_prepare
6751 && !xtrabackup_prepare_func(argv)) {
6752 return(EXIT_FAILURE);
6753 }
6754
6755 if (xtrabackup_copy_back || xtrabackup_move_back) {
6756 if (!check_if_param_set("datadir")) {
6757 mysql_data_home = get_default_datadir();
6758 }
6759 if (!copy_back())
6760 return(EXIT_FAILURE);
6761 }
6762
6763 if (xtrabackup_decrypt_decompress && !decrypt_decompress()) {
6764 return(EXIT_FAILURE);
6765 }
6766
6767 return(EXIT_SUCCESS);
6768 }
6769
6770
get_exepath(char * buf,size_t size,const char * argv0)6771 static int get_exepath(char *buf, size_t size, const char *argv0)
6772 {
6773 #ifdef _WIN32
6774 DWORD ret = GetModuleFileNameA(NULL, buf, (DWORD)size);
6775 if (ret > 0)
6776 return 0;
6777 #elif defined(__linux__)
6778 ssize_t ret = readlink("/proc/self/exe", buf, size-1);
6779 if(ret > 0)
6780 return 0;
6781 #endif
6782
6783 return my_realpath(buf, argv0, 0);
6784 }
6785
6786
6787 #if defined (__SANITIZE_ADDRESS__) && defined (__linux__)
6788 /* Avoid LeakSanitizer's false positives. */
__asan_default_options()6789 const char* __asan_default_options()
6790 {
6791 return "detect_leaks=0";
6792 }
6793 #endif
6794