1 /*
2 Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License, version 2.0, for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 #include <ndb_global.h>
26 #include <ndb_opts.h>
27 #include <Vector.hpp>
28 #include <Properties.hpp>
29 #include <ndb_limits.h>
30 #include <NdbTCP.h>
31 #include <NdbMem.h>
32 #include <NdbOut.hpp>
33 #include <OutputStream.hpp>
34 #include <NDBT_ReturnCodes.h>
35
36 #include "consumer_restore.hpp"
37 #include "consumer_printer.hpp"
38 #include "../src/ndbapi/NdbDictionaryImpl.hpp"
39
40 extern FilteredNdbOut err;
41 extern FilteredNdbOut info;
42 extern FilteredNdbOut debug;
43
44 static Uint32 g_tableCompabilityMask = 0;
45 static int ga_nodeId = 0;
46 static int ga_nParallelism = 128;
47 static int ga_backupId = 0;
48 bool ga_dont_ignore_systab_0 = false;
49 static bool ga_no_upgrade = false;
50 static bool ga_promote_attributes = false;
51 static bool ga_demote_attributes = false;
52 static Vector<class BackupConsumer *> g_consumers;
53 static BackupPrinter* g_printer = NULL;
54
55 static const char* default_backupPath = "." DIR_SEPARATOR;
56 static const char* ga_backupPath = default_backupPath;
57
58 static const char *opt_nodegroup_map_str= 0;
59 static unsigned opt_nodegroup_map_len= 0;
60 static NODE_GROUP_MAP opt_nodegroup_map[MAX_NODE_GROUP_MAPS];
61 #define OPT_NDB_NODEGROUP_MAP 'z'
62
63 const char *opt_ndb_database= NULL;
64 const char *opt_ndb_table= NULL;
65 unsigned int opt_verbose;
66 unsigned int opt_hex_format;
67 unsigned int opt_progress_frequency;
68 NDB_TICKS g_report_next;
69 Vector<BaseString> g_databases;
70 Vector<BaseString> g_tables;
71 Vector<BaseString> g_include_tables, g_exclude_tables;
72 Vector<BaseString> g_include_databases, g_exclude_databases;
73 Properties g_rewrite_databases;
74 NdbRecordPrintFormat g_ndbrecord_print_format;
75 unsigned int opt_no_binlog;
76
77 class RestoreOption
78 {
79 public:
~RestoreOption()80 virtual ~RestoreOption() { }
81 int optid;
82 BaseString argument;
83 };
84
85 Vector<class RestoreOption *> g_include_exclude;
86 static void save_include_exclude(int optid, char * argument);
87
88 static inline void parse_rewrite_database(char * argument);
89
90 /**
91 * print and restore flags
92 */
93 static bool ga_restore_epoch = false;
94 static bool ga_restore = false;
95 static bool ga_print = false;
96 static bool ga_skip_table_check = false;
97 static bool ga_exclude_missing_columns = false;
98 static int _print = 0;
99 static int _print_meta = 0;
100 static int _print_data = 0;
101 static int _print_log = 0;
102 static int _restore_data = 0;
103 static int _restore_meta = 0;
104 static int _no_restore_disk = 0;
105 static bool _preserve_trailing_spaces = false;
106 static bool ga_disable_indexes = false;
107 static bool ga_rebuild_indexes = false;
108 bool ga_skip_unknown_objects = false;
109 bool ga_skip_broken_objects = false;
110 BaseString g_options("ndb_restore");
111
112 const char *load_default_groups[]= { "mysql_cluster","ndb_restore",0 };
113
114 enum ndb_restore_options {
115 OPT_VERBOSE = NDB_STD_OPTIONS_LAST,
116 OPT_INCLUDE_TABLES,
117 OPT_EXCLUDE_TABLES,
118 OPT_INCLUDE_DATABASES,
119 OPT_EXCLUDE_DATABASES,
120 OPT_REWRITE_DATABASE
121 };
122 static const char *opt_fields_enclosed_by= NULL;
123 static const char *opt_fields_terminated_by= NULL;
124 static const char *opt_fields_optionally_enclosed_by= NULL;
125 static const char *opt_lines_terminated_by= NULL;
126
127 static const char *tab_path= NULL;
128 static int opt_append;
129 static const char *opt_exclude_tables= NULL;
130 static const char *opt_include_tables= NULL;
131 static const char *opt_exclude_databases= NULL;
132 static const char *opt_include_databases= NULL;
133 static const char *opt_rewrite_database= NULL;
134 static bool opt_restore_privilege_tables = false;
135
136 static struct my_option my_long_options[] =
137 {
138 NDB_STD_OPTS("ndb_restore"),
139 { "connect", 'c', "same as --connect-string",
140 (uchar**) &opt_ndb_connectstring, (uchar**) &opt_ndb_connectstring, 0,
141 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
142 { "nodeid", 'n', "Backup files from node with id",
143 (uchar**) &ga_nodeId, (uchar**) &ga_nodeId, 0,
144 GET_INT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
145 { "backupid", 'b', "Backup id",
146 (uchar**) &ga_backupId, (uchar**) &ga_backupId, 0,
147 GET_INT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
148 { "restore_data", 'r',
149 "Restore table data/logs into NDB Cluster using NDBAPI",
150 (uchar**) &_restore_data, (uchar**) &_restore_data, 0,
151 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
152 { "restore_meta", 'm',
153 "Restore meta data into NDB Cluster using NDBAPI",
154 (uchar**) &_restore_meta, (uchar**) &_restore_meta, 0,
155 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
156 { "no-upgrade", 'u',
157 "Don't upgrade array type for var attributes, which don't resize VAR data and don't change column attributes",
158 (uchar**) &ga_no_upgrade, (uchar**) &ga_no_upgrade, 0,
159 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
160 { "promote-attributes", 'A',
161 "Allow attributes to be promoted when restoring data from backup",
162 (uchar**) &ga_promote_attributes, (uchar**) &ga_promote_attributes, 0,
163 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
164 { "lossy-conversions", 'L',
165 "Allow lossy conversions for attributes (type demotions or integral"
166 " signed/unsigned type changes) when restoring data from backup",
167 (uchar**) &ga_demote_attributes, (uchar**) &ga_demote_attributes, 0,
168 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
169 { "preserve-trailing-spaces", 'P',
170 "Allow to preserve the tailing spaces (including paddings) When char->varchar or binary->varbinary is promoted",
171 (uchar**) &_preserve_trailing_spaces, (uchar**)_preserve_trailing_spaces , 0,
172 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
173 { "no-restore-disk-objects", 'd',
174 "Dont restore disk objects (tablespace/logfilegroups etc)",
175 (uchar**) &_no_restore_disk, (uchar**) &_no_restore_disk, 0,
176 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
177 { "restore_epoch", 'e',
178 "Restore epoch info into the status table. Convenient on a MySQL Cluster "
179 "replication slave, for starting replication. The row in "
180 NDB_REP_DB "." NDB_APPLY_TABLE " with id 0 will be updated/inserted.",
181 (uchar**) &ga_restore_epoch, (uchar**) &ga_restore_epoch, 0,
182 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
183 { "skip-table-check", 's', "Skip table structure check during restore of data",
184 (uchar**) &ga_skip_table_check, (uchar**) &ga_skip_table_check, 0,
185 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
186 { "parallelism", 'p',
187 "No of parallel transactions during restore of data."
188 "(parallelism can be 1 to 1024)",
189 (uchar**) &ga_nParallelism, (uchar**) &ga_nParallelism, 0,
190 GET_INT, REQUIRED_ARG, 128, 1, 1024, 0, 1, 0 },
191 { "print", NDB_OPT_NOSHORT, "Print metadata, data and log to stdout",
192 (uchar**) &_print, (uchar**) &_print, 0,
193 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
194 { "print_data", NDB_OPT_NOSHORT, "Print data to stdout",
195 (uchar**) &_print_data, (uchar**) &_print_data, 0,
196 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
197 { "print_meta", NDB_OPT_NOSHORT, "Print meta data to stdout",
198 (uchar**) &_print_meta, (uchar**) &_print_meta, 0,
199 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
200 { "print_log", NDB_OPT_NOSHORT, "Print log to stdout",
201 (uchar**) &_print_log, (uchar**) &_print_log, 0,
202 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
203 { "backup_path", NDB_OPT_NOSHORT, "Path to backup files",
204 (uchar**) &ga_backupPath, (uchar**) &ga_backupPath, 0,
205 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
206 { "dont_ignore_systab_0", 'f',
207 "Do not ignore system table during --print-data.",
208 (uchar**) &ga_dont_ignore_systab_0, (uchar**) &ga_dont_ignore_systab_0, 0,
209 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
210 { "ndb-nodegroup-map", OPT_NDB_NODEGROUP_MAP,
211 "Nodegroup map for ndbcluster. Syntax: list of (source_ng, dest_ng)",
212 (uchar**) &opt_nodegroup_map_str,
213 (uchar**) &opt_nodegroup_map_str,
214 0,
215 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
216 { "fields-enclosed-by", NDB_OPT_NOSHORT,
217 "Fields are enclosed by ...",
218 (uchar**) &opt_fields_enclosed_by, (uchar**) &opt_fields_enclosed_by, 0,
219 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
220 { "fields-terminated-by", NDB_OPT_NOSHORT,
221 "Fields are terminated by ...",
222 (uchar**) &opt_fields_terminated_by,
223 (uchar**) &opt_fields_terminated_by, 0,
224 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
225 { "fields-optionally-enclosed-by", NDB_OPT_NOSHORT,
226 "Fields are optionally enclosed by ...",
227 (uchar**) &opt_fields_optionally_enclosed_by,
228 (uchar**) &opt_fields_optionally_enclosed_by, 0,
229 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
230 { "hex", NDB_OPT_NOSHORT, "print binary types in hex format",
231 (uchar**) &opt_hex_format, (uchar**) &opt_hex_format, 0,
232 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
233 { "tab", 'T', "Creates tab separated textfile for each table to "
234 "given path. (creates .txt files)",
235 (uchar**) &tab_path, (uchar**) &tab_path, 0,
236 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
237 { "append", NDB_OPT_NOSHORT, "for --tab append data to file",
238 (uchar**) &opt_append, (uchar**) &opt_append, 0,
239 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
240 { "lines-terminated-by", NDB_OPT_NOSHORT, "",
241 (uchar**) &opt_lines_terminated_by, (uchar**) &opt_lines_terminated_by, 0,
242 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
243 { "progress-frequency", NDB_OPT_NOSHORT,
244 "Print status uf restore periodically in given seconds",
245 (uchar**) &opt_progress_frequency, (uchar**) &opt_progress_frequency, 0,
246 GET_INT, REQUIRED_ARG, 0, 0, 65535, 0, 0, 0 },
247 { "no-binlog", NDB_OPT_NOSHORT,
248 "If a mysqld is connected and has binary log, do not log the restored data",
249 (uchar**) &opt_no_binlog, (uchar**) &opt_no_binlog, 0,
250 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
251 { "verbose", OPT_VERBOSE,
252 "verbosity",
253 (uchar**) &opt_verbose, (uchar**) &opt_verbose, 0,
254 GET_INT, REQUIRED_ARG, 1, 0, 255, 0, 0, 0 },
255 { "include-databases", OPT_INCLUDE_DATABASES,
256 "Comma separated list of databases to restore. Example: db1,db3",
257 (uchar**) &opt_include_databases, (uchar**) &opt_include_databases, 0,
258 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
259 { "exclude-databases", OPT_EXCLUDE_DATABASES,
260 "Comma separated list of databases to not restore. Example: db1,db3",
261 (uchar**) &opt_exclude_databases, (uchar**) &opt_exclude_databases, 0,
262 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
263 { "rewrite-database", OPT_REWRITE_DATABASE,
264 "A pair 'source,dest' of database names from/into which to restore. "
265 "Example: --rewrite-database=oldDb,newDb",
266 (uchar**) &opt_rewrite_database, (uchar**) &opt_rewrite_database, 0,
267 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
268 { "include-tables", OPT_INCLUDE_TABLES, "Comma separated list of tables to "
269 "restore. Table name should include database name. Example: db1.t1,db3.t1",
270 (uchar**) &opt_include_tables, (uchar**) &opt_include_tables, 0,
271 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
272 { "exclude-tables", OPT_EXCLUDE_TABLES, "Comma separated list of tables to "
273 "not restore. Table name should include database name. "
274 "Example: db1.t1,db3.t1",
275 (uchar**) &opt_exclude_tables, (uchar**) &opt_exclude_tables, 0,
276 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
277 { "restore-privilege-tables", NDB_OPT_NOSHORT,
278 "Restore privilege tables (after they have been moved to ndb)",
279 (uchar**) &opt_restore_privilege_tables,
280 (uchar**) &opt_restore_privilege_tables, 0,
281 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
282 { "exclude-missing-columns", NDB_OPT_NOSHORT,
283 "Ignore columns present in backup but not in database",
284 (uchar**) &ga_exclude_missing_columns,
285 (uchar**) &ga_exclude_missing_columns, 0,
286 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
287 { "disable-indexes", NDB_OPT_NOSHORT,
288 "Disable indexes",
289 (uchar**) &ga_disable_indexes,
290 (uchar**) &ga_disable_indexes, 0,
291 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
292 { "rebuild-indexes", NDB_OPT_NOSHORT,
293 "Rebuild indexes",
294 (uchar**) &ga_rebuild_indexes,
295 (uchar**) &ga_rebuild_indexes, 0,
296 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
297 { "skip-unknown-objects", 256, "Skip unknown object when parsing backup",
298 (uchar**) &ga_skip_unknown_objects, (uchar**) &ga_skip_unknown_objects, 0,
299 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
300 { "skip-broken-objects", 256, "Skip broken object when parsing backup",
301 (uchar**) &ga_skip_broken_objects, (uchar**) &ga_skip_broken_objects, 0,
302 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
303 { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
304 };
305
306
analyse_one_map(char * map_str,uint16 * source,uint16 * dest)307 static char* analyse_one_map(char *map_str, uint16 *source, uint16 *dest)
308 {
309 char *end_ptr;
310 int number;
311 DBUG_ENTER("analyse_one_map");
312 /*
313 Search for pattern ( source_ng , dest_ng )
314 */
315
316 while (isspace(*map_str)) map_str++;
317
318 if (*map_str != '(')
319 {
320 DBUG_RETURN(NULL);
321 }
322 map_str++;
323
324 while (isspace(*map_str)) map_str++;
325
326 number= strtol(map_str, &end_ptr, 10);
327 if (!end_ptr || number < 0 || number >= MAX_NODE_GROUP_MAPS)
328 {
329 DBUG_RETURN(NULL);
330 }
331 *source= (uint16)number;
332 map_str= end_ptr;
333
334 while (isspace(*map_str)) map_str++;
335
336 if (*map_str != ',')
337 {
338 DBUG_RETURN(NULL);
339 }
340 map_str++;
341
342 number= strtol(map_str, &end_ptr, 10);
343 if (!end_ptr || number < 0 || number >= NDB_UNDEF_NODEGROUP)
344 {
345 DBUG_RETURN(NULL);
346 }
347 *dest= (uint16)number;
348 map_str= end_ptr;
349
350 if (*map_str != ')')
351 {
352 DBUG_RETURN(NULL);
353 }
354 map_str++;
355
356 while (isspace(*map_str)) map_str++;
357 DBUG_RETURN(map_str);
358 }
359
insert_ng_map(NODE_GROUP_MAP * ng_map,uint16 source_ng,uint16 dest_ng)360 static bool insert_ng_map(NODE_GROUP_MAP *ng_map,
361 uint16 source_ng, uint16 dest_ng)
362 {
363 uint index= source_ng;
364 uint ng_index= ng_map[index].no_maps;
365
366 opt_nodegroup_map_len++;
367 if (ng_index >= MAX_MAPS_PER_NODE_GROUP)
368 return true;
369 ng_map[index].no_maps++;
370 ng_map[index].map_array[ng_index]= dest_ng;
371 return false;
372 }
373
init_nodegroup_map()374 static void init_nodegroup_map()
375 {
376 uint i,j;
377 NODE_GROUP_MAP *ng_map = &opt_nodegroup_map[0];
378
379 for (i = 0; i < MAX_NODE_GROUP_MAPS; i++)
380 {
381 ng_map[i].no_maps= 0;
382 for (j= 0; j < MAX_MAPS_PER_NODE_GROUP; j++)
383 ng_map[i].map_array[j]= NDB_UNDEF_NODEGROUP;
384 }
385 }
386
analyse_nodegroup_map(const char * ng_map_str,NODE_GROUP_MAP * ng_map)387 static bool analyse_nodegroup_map(const char *ng_map_str,
388 NODE_GROUP_MAP *ng_map)
389 {
390 uint16 source_ng, dest_ng;
391 char *local_str= (char*)ng_map_str;
392 DBUG_ENTER("analyse_nodegroup_map");
393
394 do
395 {
396 if (!local_str)
397 {
398 DBUG_RETURN(TRUE);
399 }
400 local_str= analyse_one_map(local_str, &source_ng, &dest_ng);
401 if (!local_str)
402 {
403 DBUG_RETURN(TRUE);
404 }
405 if (insert_ng_map(ng_map, source_ng, dest_ng))
406 {
407 DBUG_RETURN(TRUE);
408 }
409 if (!(*local_str))
410 break;
411 } while (TRUE);
412 DBUG_RETURN(FALSE);
413 }
414
short_usage_sub(void)415 static void short_usage_sub(void)
416 {
417 ndb_short_usage_sub("[<path to backup files>]");
418 }
usage()419 static void usage()
420 {
421 ndb_usage(short_usage_sub, load_default_groups, my_long_options);
422 }
423
424 static my_bool
get_one_option(int optid,const struct my_option * opt MY_ATTRIBUTE ((unused)),char * argument)425 get_one_option(int optid, const struct my_option *opt MY_ATTRIBUTE((unused)),
426 char *argument)
427 {
428 #ifndef DBUG_OFF
429 opt_debug= "d:t:O,/tmp/ndb_restore.trace";
430 #endif
431 ndb_std_get_one_option(optid, opt, argument);
432 switch (optid) {
433 case OPT_VERBOSE:
434 info.setThreshold(255-opt_verbose);
435 break;
436 case 'n':
437 if (ga_nodeId == 0)
438 {
439 err << "Error in --nodeid,-n setting, see --help";
440 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
441 }
442 info.setLevel(254);
443 info << "Nodeid = " << ga_nodeId << endl;
444 break;
445 case 'b':
446 if (ga_backupId == 0)
447 {
448 err << "Error in --backupid,-b setting, see --help";
449 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
450 }
451 info.setLevel(254);
452 info << "Backup Id = " << ga_backupId << endl;
453 break;
454 case OPT_NDB_NODEGROUP_MAP:
455 /*
456 This option is used to set a map from nodegroup in original cluster
457 to nodegroup in new cluster.
458 */
459 opt_nodegroup_map_len= 0;
460
461 info.setLevel(254);
462 info << "Analyse node group map" << endl;
463 if (analyse_nodegroup_map(opt_nodegroup_map_str,
464 &opt_nodegroup_map[0]))
465 {
466 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
467 }
468 break;
469 case OPT_INCLUDE_DATABASES:
470 case OPT_EXCLUDE_DATABASES:
471 case OPT_INCLUDE_TABLES:
472 case OPT_EXCLUDE_TABLES:
473 save_include_exclude(optid, argument);
474 break;
475 case OPT_REWRITE_DATABASE:
476 parse_rewrite_database(argument);
477 break;
478 }
479 return 0;
480 }
481
482 static const char* SCHEMA_NAME="/def/";
483 static const int SCHEMA_NAME_SIZE= 5;
484
485 int
makeInternalTableName(const BaseString & externalName,BaseString & internalName)486 makeInternalTableName(const BaseString &externalName,
487 BaseString& internalName)
488 {
489 // Make dbname.table1 into dbname/def/table1
490 Vector<BaseString> parts;
491
492 // Must contain a dot
493 if (externalName.indexOf('.') == -1)
494 return -1;
495 externalName.split(parts,".");
496 // .. and only 1 dot
497 if (parts.size() != 2)
498 return -1;
499 internalName.clear();
500 internalName.append(parts[0]); // db name
501 internalName.append(SCHEMA_NAME); // /def/
502 internalName.append(parts[1]); // table name
503 return 0;
504 }
505
506 void
processTableList(const char * str,Vector<BaseString> & lst)507 processTableList(const char* str, Vector<BaseString> &lst)
508 {
509 // Process tables list like db1.t1,db2.t1 and exits when
510 // it finds problems.
511 Vector<BaseString> tmp;
512 unsigned int i;
513 /* Split passed string on comma into 2 BaseStrings in the vector */
514 BaseString(str).split(tmp,",");
515 for (i=0; i < tmp.size(); i++)
516 {
517 BaseString internalName;
518 if (makeInternalTableName(tmp[i], internalName))
519 {
520 info << "`" << tmp[i] << "` is not a valid tablename!" << endl;
521 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
522 }
523 lst.push_back(internalName);
524 }
525 }
526
527 BaseString
makeExternalTableName(const BaseString & internalName)528 makeExternalTableName(const BaseString &internalName)
529 {
530 // Make dbname/def/table1 into dbname.table1
531 BaseString externalName;
532
533 ssize_t idx = internalName.indexOf('/');
534 externalName = internalName.substr(0,idx);
535 externalName.append(".");
536 externalName.append(internalName.substr(idx + SCHEMA_NAME_SIZE,
537 internalName.length()));
538 return externalName;
539 }
540
541 #include "../../../../sql/ndb_dist_priv_util.h"
542
543 // Exclude privilege tables unless explicitely included
544 void
exclude_privilege_tables()545 exclude_privilege_tables()
546 {
547 const char* table_name;
548 Ndb_dist_priv_util dist_priv;
549 while((table_name= dist_priv.iter_next_table()))
550 {
551 BaseString priv_tab;
552 priv_tab.assfmt("%s.%s", dist_priv.database(), table_name);
553 g_exclude_tables.push_back(priv_tab);
554 save_include_exclude(OPT_EXCLUDE_TABLES, (char *)priv_tab.c_str());
555 }
556 }
557
558
559 bool
readArguments(int * pargc,char *** pargv)560 readArguments(int *pargc, char*** pargv)
561 {
562 Uint32 i;
563 BaseString tmp;
564 debug << "Load defaults" << endl;
565 const char *load_default_groups[]= { "mysql_cluster","ndb_restore",0 };
566
567 init_nodegroup_map();
568 load_defaults("my",load_default_groups,pargc,pargv);
569 debug << "handle_options" << endl;
570
571 ndb_opt_set_usage_funcs(short_usage_sub, usage);
572
573 if (handle_options(pargc, pargv, my_long_options, get_one_option))
574 {
575 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
576 }
577 for (i = 0; i < MAX_NODE_GROUP_MAPS; i++)
578 opt_nodegroup_map[i].curr_index = 0;
579
580 #if 0
581 /*
582 Test code written t{
583 o verify nodegroup mapping
584 */
585 printf("Handled options successfully\n");
586 Uint16 map_ng[16];
587 Uint32 j;
588 for (j = 0; j < 4; j++)
589 {
590 for (i = 0; i < 4 ; i++)
591 map_ng[i] = i;
592 map_nodegroups(&map_ng[0], (Uint32)4);
593 for (i = 0; i < 4 ; i++)
594 printf("NG %u mapped to %u \n", i, map_ng[i]);
595 }
596 for (j = 0; j < 4; j++)
597 {
598 for (i = 0; i < 8 ; i++)
599 map_ng[i] = i >> 1;
600 map_nodegroups(&map_ng[0], (Uint32)8);
601 for (i = 0; i < 8 ; i++)
602 printf("NG %u mapped to %u \n", i >> 1, map_ng[i]);
603 }
604 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
605 #endif
606
607 g_printer = new BackupPrinter(opt_nodegroup_map,
608 opt_nodegroup_map_len);
609 if (g_printer == NULL)
610 return false;
611
612 BackupRestore* restore = new BackupRestore(opt_ndb_connectstring,
613 opt_ndb_nodeid,
614 opt_nodegroup_map,
615 opt_nodegroup_map_len,
616 ga_nParallelism);
617 if (restore == NULL)
618 {
619 delete g_printer;
620 g_printer = NULL;
621 return false;
622 }
623
624 if (_print)
625 {
626 ga_print = true;
627 ga_restore = true;
628 g_printer->m_print = true;
629 }
630 if (_print_meta)
631 {
632 ga_print = true;
633 g_printer->m_print_meta = true;
634 }
635 if (_print_data)
636 {
637 ga_print = true;
638 g_printer->m_print_data = true;
639 }
640 if (_print_log)
641 {
642 ga_print = true;
643 g_printer->m_print_log = true;
644 }
645
646 if (_restore_data)
647 {
648 ga_restore = true;
649 restore->m_restore = true;
650 }
651
652 if (_restore_meta)
653 {
654 // ga_restore = true;
655 restore->m_restore_meta = true;
656 }
657
658 if (_no_restore_disk)
659 {
660 restore->m_no_restore_disk = true;
661 }
662
663 if (ga_no_upgrade)
664 {
665 restore->m_no_upgrade = true;
666 }
667
668 if (_preserve_trailing_spaces)
669 {
670 restore->m_preserve_trailing_spaces = true;
671 }
672
673 if (ga_restore_epoch)
674 {
675 restore->m_restore_epoch = true;
676 }
677
678 if (ga_disable_indexes)
679 {
680 restore->m_disable_indexes = true;
681 }
682
683 if (ga_rebuild_indexes)
684 {
685 restore->m_rebuild_indexes = true;
686 }
687
688 {
689 BackupConsumer * c = g_printer;
690 g_consumers.push_back(c);
691 }
692 {
693 BackupConsumer * c = restore;
694 g_consumers.push_back(c);
695 }
696 for (;;)
697 {
698 int i= 0;
699 if (ga_backupPath == default_backupPath)
700 {
701 // Set backup file path
702 if ((*pargv)[i] == NULL)
703 break;
704 ga_backupPath = (*pargv)[i++];
705 }
706 if ((*pargv)[i] == NULL)
707 break;
708 g_databases.push_back((*pargv)[i++]);
709 while ((*pargv)[i] != NULL)
710 {
711 g_tables.push_back((*pargv)[i++]);
712 }
713 break;
714 }
715 info.setLevel(254);
716 info << "backup path = " << ga_backupPath << endl;
717 if (g_databases.size() > 0)
718 {
719 info << "WARNING! Using deprecated syntax for selective object restoration." << endl;
720 info << "Please use --include-*/--exclude-* options in future." << endl;
721 info << "Restoring only from database " << g_databases[0].c_str() << endl;
722 if (g_tables.size() > 0)
723 {
724 info << "Restoring tables:";
725 }
726 for (unsigned i= 0; i < g_tables.size(); i++)
727 {
728 info << " " << g_tables[i].c_str();
729 }
730 if (g_tables.size() > 0)
731 info << endl;
732 }
733
734 if (ga_restore)
735 {
736 // Exclude privilege tables unless explicitely included
737 if (!opt_restore_privilege_tables)
738 exclude_privilege_tables();
739
740 // Move over old style arguments to include/exclude lists
741 if (g_databases.size() > 0)
742 {
743 BaseString tab_prefix, tab;
744 tab_prefix.append(g_databases[0].c_str());
745 tab_prefix.append(".");
746 if (g_tables.size() == 0)
747 {
748 g_include_databases.push_back(g_databases[0]);
749 save_include_exclude(OPT_INCLUDE_DATABASES,
750 (char *)g_databases[0].c_str());
751 }
752 for (unsigned i= 0; i < g_tables.size(); i++)
753 {
754 tab.assign(tab_prefix);
755 tab.append(g_tables[i]);
756 g_include_tables.push_back(tab);
757 save_include_exclude(OPT_INCLUDE_TABLES, (char *)tab.c_str());
758 }
759 }
760 }
761
762 if (opt_include_databases)
763 {
764 tmp = BaseString(opt_include_databases);
765 tmp.split(g_include_databases,",");
766 info << "Including Databases: ";
767 for (i= 0; i < g_include_databases.size(); i++)
768 {
769 info << g_include_databases[i] << " ";
770 }
771 info << endl;
772 }
773
774 if (opt_exclude_databases)
775 {
776 tmp = BaseString(opt_exclude_databases);
777 tmp.split(g_exclude_databases,",");
778 info << "Excluding databases: ";
779 for (i= 0; i < g_exclude_databases.size(); i++)
780 {
781 info << g_exclude_databases[i] << " ";
782 }
783 info << endl;
784 }
785
786 if (opt_rewrite_database)
787 {
788 info << "Rewriting databases:";
789 Properties::Iterator it(&g_rewrite_databases);
790 const char * src;
791 for (src = it.first(); src != NULL; src = it.next()) {
792 const char * dst = NULL;
793 bool r = g_rewrite_databases.get(src, &dst);
794 assert(r && (dst != NULL));
795 info << " (" << src << "->" << dst << ")";
796 }
797 info << endl;
798 }
799
800 if (opt_include_tables)
801 {
802 processTableList(opt_include_tables, g_include_tables);
803 info << "Including tables: ";
804 for (i= 0; i < g_include_tables.size(); i++)
805 {
806 info << makeExternalTableName(g_include_tables[i]).c_str() << " ";
807 }
808 info << endl;
809 }
810
811 if (opt_exclude_tables)
812 {
813 processTableList(opt_exclude_tables, g_exclude_tables);
814 info << "Excluding tables: ";
815 for (i= 0; i < g_exclude_tables.size(); i++)
816 {
817 info << makeExternalTableName(g_exclude_tables[i]).c_str() << " ";
818 }
819 info << endl;
820 }
821
822 /*
823 the below formatting follows the formatting from mysqldump
824 do not change unless to adopt to changes in mysqldump
825 */
826 g_ndbrecord_print_format.fields_enclosed_by=
827 opt_fields_enclosed_by ? opt_fields_enclosed_by : "";
828 g_ndbrecord_print_format.fields_terminated_by=
829 opt_fields_terminated_by ? opt_fields_terminated_by : "\t";
830 g_ndbrecord_print_format.fields_optionally_enclosed_by=
831 opt_fields_optionally_enclosed_by ? opt_fields_optionally_enclosed_by : "";
832 g_ndbrecord_print_format.lines_terminated_by=
833 opt_lines_terminated_by ? opt_lines_terminated_by : "\n";
834 if (g_ndbrecord_print_format.fields_optionally_enclosed_by[0] == '\0')
835 g_ndbrecord_print_format.null_string= "\\N";
836 else
837 g_ndbrecord_print_format.null_string= "";
838 g_ndbrecord_print_format.hex_prefix= "";
839 g_ndbrecord_print_format.hex_format= opt_hex_format;
840
841 if (ga_skip_table_check)
842 {
843 g_tableCompabilityMask = ~(Uint32)0;
844 ga_skip_unknown_objects = true;
845 }
846
847 if (ga_promote_attributes)
848 {
849 g_tableCompabilityMask |= TCM_ATTRIBUTE_PROMOTION;
850 }
851
852 if (ga_demote_attributes)
853 {
854 g_tableCompabilityMask |= TCM_ATTRIBUTE_DEMOTION;
855 }
856
857 if (ga_exclude_missing_columns)
858 {
859 g_tableCompabilityMask |= TCM_EXCLUDE_MISSING_COLUMNS;
860 }
861 return true;
862 }
863
864 void
clearConsumers()865 clearConsumers()
866 {
867 for(Uint32 i= 0; i<g_consumers.size(); i++)
868 delete g_consumers[i];
869 g_consumers.clear();
870 }
871
872 static inline bool
checkSysTable(const TableS * table)873 checkSysTable(const TableS* table)
874 {
875 return ! table->getSysTable();
876 }
877
878 static inline bool
checkSysTable(const RestoreMetaData & metaData,uint i)879 checkSysTable(const RestoreMetaData& metaData, uint i)
880 {
881 assert(i < metaData.getNoOfTables());
882 return checkSysTable(metaData[i]);
883 }
884
885 static inline bool
isBlobTable(const TableS * table)886 isBlobTable(const TableS* table)
887 {
888 return table->getMainTable() != NULL;
889 }
890
891 static inline bool
isIndex(const TableS * table)892 isIndex(const TableS* table)
893 {
894 const NdbTableImpl & tmptab = NdbTableImpl::getImpl(* table->m_dictTable);
895 return (int) tmptab.m_indexType != (int) NdbDictionary::Index::Undefined;
896 }
897
898 static inline bool
isSYSTAB_0(const TableS * table)899 isSYSTAB_0(const TableS* table)
900 {
901 return table->isSYSTAB_0();
902 }
903
904 static inline bool
isInList(BaseString & needle,Vector<BaseString> & lst)905 isInList(BaseString &needle, Vector<BaseString> &lst){
906 unsigned int i= 0;
907 for (i= 0; i < lst.size(); i++)
908 {
909 if (strcmp(needle.c_str(), lst[i].c_str()) == 0)
910 return true;
911 }
912 return false;
913 }
914
915 const char*
getTableName(const TableS * table)916 getTableName(const TableS* table)
917 {
918 const char *table_name;
919 if (isBlobTable(table))
920 table_name= table->getMainTable()->getTableName();
921 else if (isIndex(table))
922 table_name=
923 NdbTableImpl::getImpl(*table->m_dictTable).m_primaryTable.c_str();
924 else
925 table_name= table->getTableName();
926
927 return table_name;
928 }
929
parse_rewrite_database(char * argument)930 static void parse_rewrite_database(char * argument)
931 {
932 const BaseString arg(argument);
933 Vector<BaseString> args;
934 unsigned int n = arg.split(args, ",");
935 if ((n == 2)
936 && (args[0].length() > 0)
937 && (args[1].length() > 0)) {
938 const BaseString src = args[0];
939 const BaseString dst = args[1];
940 const bool replace = true;
941 bool r = g_rewrite_databases.put(src.c_str(), dst.c_str(), replace);
942 assert(r);
943 return; // ok
944 }
945
946 info << "argument `" << arg.c_str()
947 << "` is not a pair 'a,b' of non-empty names." << endl;
948 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
949 }
950
save_include_exclude(int optid,char * argument)951 static void save_include_exclude(int optid, char * argument)
952 {
953 BaseString arg = argument;
954 Vector<BaseString> args;
955 arg.split(args, ",");
956 for (uint i = 0; i < args.size(); i++)
957 {
958 RestoreOption * option = new RestoreOption();
959 BaseString arg;
960
961 option->optid = optid;
962 switch (optid) {
963 case OPT_INCLUDE_TABLES:
964 case OPT_EXCLUDE_TABLES:
965 if (makeInternalTableName(args[i], arg))
966 {
967 info << "`" << args[i] << "` is not a valid tablename!" << endl;
968 exit(NDBT_ProgramExit(NDBT_WRONGARGS));
969 }
970 break;
971 default:
972 arg = args[i];
973 break;
974 }
975 option->argument = arg;
976 g_include_exclude.push_back(option);
977 }
978 }
check_include_exclude(BaseString database,BaseString table)979 static bool check_include_exclude(BaseString database, BaseString table)
980 {
981 const char * db = database.c_str();
982 const char * tbl = table.c_str();
983 bool do_include = true;
984
985 if (g_include_databases.size() != 0 ||
986 g_include_tables.size() != 0)
987 {
988 /*
989 User has explicitly specified what databases
990 and/or tables should be restored. If no match is
991 found then DON'T restore table.
992 */
993 do_include = false;
994 }
995 if (do_include &&
996 (g_exclude_databases.size() != 0 ||
997 g_exclude_tables.size() != 0))
998 {
999 /*
1000 User has not explicitly specified what databases
1001 and/or tables should be restored.
1002 User has explicitly specified what databases
1003 and/or tables should NOT be restored. If no match is
1004 found then DO restore table.
1005 */
1006 do_include = true;
1007 }
1008
1009 if (g_include_exclude.size() != 0)
1010 {
1011 /*
1012 Scan include exclude arguments in reverse.
1013 First matching include causes table to be restored.
1014 first matching exclude causes table NOT to be restored.
1015 */
1016 for(uint i = g_include_exclude.size(); i > 0; i--)
1017 {
1018 RestoreOption *option = g_include_exclude[i-1];
1019 switch (option->optid) {
1020 case OPT_INCLUDE_TABLES:
1021 if (strcmp(tbl, option->argument.c_str()) == 0)
1022 return true; // do include
1023 break;
1024 case OPT_EXCLUDE_TABLES:
1025 if (strcmp(tbl, option->argument.c_str()) == 0)
1026 return false; // don't include
1027 break;
1028 case OPT_INCLUDE_DATABASES:
1029 if (strcmp(db, option->argument.c_str()) == 0)
1030 return true; // do include
1031 break;
1032 case OPT_EXCLUDE_DATABASES:
1033 if (strcmp(db, option->argument.c_str()) == 0)
1034 return false; // don't include
1035 break;
1036 default:
1037 continue;
1038 }
1039 }
1040 }
1041
1042 return do_include;
1043 }
1044
1045 static inline bool
checkDoRestore(const TableS * table)1046 checkDoRestore(const TableS* table)
1047 {
1048 bool ret = true;
1049 BaseString db, tbl;
1050
1051 tbl.assign(getTableName(table));
1052 ssize_t idx = tbl.indexOf('/');
1053 db = tbl.substr(0, idx);
1054
1055 /*
1056 Include/exclude flags are evaluated right
1057 to left, and first match overrides any other
1058 matches. Non-overlapping arguments are accumulative.
1059 If no include flags are specified this means all databases/tables
1060 except any excluded are restored.
1061 If include flags are specified than only those databases
1062 or tables specified are restored.
1063 */
1064 ret = check_include_exclude(db, tbl);
1065 return ret;
1066 }
1067
1068 static inline bool
checkDbAndTableName(const TableS * table)1069 checkDbAndTableName(const TableS* table)
1070 {
1071 if (table->isBroken())
1072 return false;
1073
1074 // If new options are given, ignore the old format
1075 if (opt_include_tables || g_exclude_tables.size() > 0 ||
1076 opt_include_databases || opt_exclude_databases ) {
1077 return (checkDoRestore(table));
1078 }
1079
1080 if (g_tables.size() == 0 && g_databases.size() == 0)
1081 return true;
1082
1083 if (g_databases.size() == 0)
1084 g_databases.push_back("TEST_DB");
1085
1086 // Filter on the main table name for indexes and blobs
1087 const char *table_name= getTableName(table);
1088
1089 unsigned i;
1090 for (i= 0; i < g_databases.size(); i++)
1091 {
1092 if (strncmp(table_name, g_databases[i].c_str(),
1093 g_databases[i].length()) == 0 &&
1094 table_name[g_databases[i].length()] == '/')
1095 {
1096 // we have a match
1097 if (g_databases.size() > 1 || g_tables.size() == 0)
1098 return true;
1099 break;
1100 }
1101 }
1102 if (i == g_databases.size())
1103 return false; // no match found
1104
1105 while (*table_name != '/') table_name++;
1106 table_name++;
1107 while (*table_name != '/') table_name++;
1108 table_name++;
1109
1110 // Check if table should be restored
1111 for (i= 0; i < g_tables.size(); i++)
1112 {
1113 if (strcmp(table_name, g_tables[i].c_str()) == 0)
1114 return true;
1115 }
1116 return false;
1117 }
1118
1119 static void
free_data_callback()1120 free_data_callback()
1121 {
1122 for(Uint32 i= 0; i < g_consumers.size(); i++)
1123 g_consumers[i]->tuple_free();
1124 }
1125
exitHandler(int code)1126 static void exitHandler(int code)
1127 {
1128 NDBT_ProgramExit(code);
1129 if (opt_core)
1130 abort();
1131 else
1132 exit(code);
1133 }
1134
init_progress()1135 static void init_progress()
1136 {
1137 Uint64 now = NdbTick_CurrentMillisecond() / 1000;
1138 g_report_next = now + opt_progress_frequency;
1139 }
1140
check_progress()1141 static int check_progress()
1142 {
1143 if (!opt_progress_frequency)
1144 return 0;
1145
1146 NDB_TICKS now = NdbTick_CurrentMillisecond() / 1000;
1147
1148 if (now >= g_report_next)
1149 {
1150 g_report_next = now + opt_progress_frequency;
1151 return 1;
1152 }
1153 return 0;
1154 }
1155
report_progress(const char * prefix,const BackupFile & f)1156 static void report_progress(const char *prefix, const BackupFile &f)
1157 {
1158 info.setLevel(255);
1159 if (f.get_file_size())
1160 info << prefix << (f.get_file_pos() * 100 + f.get_file_size()-1) / f.get_file_size()
1161 << "%(" << f.get_file_pos() << " bytes)\n";
1162 else
1163 info << prefix << f.get_file_pos() << " bytes\n";
1164 }
1165
1166 /**
1167 * Reports, clears information on columns where data truncation was detected.
1168 */
1169 static void
check_data_truncations(const TableS * table)1170 check_data_truncations(const TableS * table)
1171 {
1172 assert(table);
1173 const char * tname = table->getTableName();
1174 const int n = table->getNoOfAttributes();
1175 for (int i = 0; i < n; i++) {
1176 AttributeDesc * desc = table->getAttributeDesc(i);
1177 if (desc->truncation_detected) {
1178 const char * cname = desc->m_column->getName();
1179 info.setLevel(254);
1180 info << "Data truncation(s) detected for attribute: "
1181 << tname << "." << cname << endl;
1182 desc->truncation_detected = false;
1183 }
1184 }
1185 }
1186
1187 int
main(int argc,char ** argv)1188 main(int argc, char** argv)
1189 {
1190 NDB_INIT(argv[0]);
1191
1192 debug << "Start readArguments" << endl;
1193 if (!readArguments(&argc, &argv))
1194 {
1195 exitHandler(NDBT_FAILED);
1196 }
1197
1198 g_options.appfmt(" -b %u", ga_backupId);
1199 g_options.appfmt(" -n %d", ga_nodeId);
1200 if (_restore_meta)
1201 g_options.appfmt(" -m");
1202 if (ga_no_upgrade)
1203 g_options.appfmt(" -u");
1204 if (ga_promote_attributes)
1205 g_options.appfmt(" -A");
1206 if (ga_demote_attributes)
1207 g_options.appfmt(" -L");
1208 if (_preserve_trailing_spaces)
1209 g_options.appfmt(" -P");
1210 if (ga_skip_table_check)
1211 g_options.appfmt(" -s");
1212 if (_restore_data)
1213 g_options.appfmt(" -r");
1214 if (ga_restore_epoch)
1215 g_options.appfmt(" -e");
1216 if (_no_restore_disk)
1217 g_options.appfmt(" -d");
1218 if (ga_exclude_missing_columns)
1219 g_options.append(" --exclude-missing-columns");
1220 if (ga_disable_indexes)
1221 g_options.append(" --disable-indexes");
1222 if (ga_rebuild_indexes)
1223 g_options.append(" --rebuild-indexes");
1224 g_options.appfmt(" -p %d", ga_nParallelism);
1225 if (ga_skip_unknown_objects)
1226 g_options.append(" --skip-unknown-objects");
1227 if (ga_skip_broken_objects)
1228 g_options.append(" --skip-broken-objects");
1229
1230 init_progress();
1231
1232 /**
1233 * we must always load meta data, even if we will only print it to stdout
1234 */
1235 debug << "Start restoring meta data" << endl;
1236 RestoreMetaData metaData(ga_backupPath, ga_nodeId, ga_backupId);
1237 if (!metaData.readHeader())
1238 {
1239 err << "Failed to read " << metaData.getFilename() << endl << endl;
1240 exitHandler(NDBT_FAILED);
1241 }
1242
1243 const BackupFormat::FileHeader & tmp = metaData.getFileHeader();
1244 const Uint32 version = tmp.BackupVersion;
1245
1246 char buf[NDB_VERSION_STRING_BUF_SZ];
1247 info.setLevel(254);
1248 info << "Backup version in files: "
1249 << ndbGetVersionString(version, 0,
1250 isDrop6(version) ? "-drop6" : 0,
1251 buf, sizeof(buf));
1252 if (version >= NDBD_RAW_LCP)
1253 {
1254 info << " ndb version: "
1255 << ndbGetVersionString(tmp.NdbVersion, tmp.MySQLVersion, 0,
1256 buf, sizeof(buf));
1257 }
1258
1259 info << endl;
1260
1261 /**
1262 * check wheater we can restore the backup (right version).
1263 */
1264 // in these versions there was an error in how replica info was
1265 // stored on disk
1266 if (version >= MAKE_VERSION(5,1,3) && version <= MAKE_VERSION(5,1,9))
1267 {
1268 err << "Restore program incompatible with backup versions between "
1269 << ndbGetVersionString(MAKE_VERSION(5,1,3), 0, 0, buf, sizeof(buf))
1270 << " and "
1271 << ndbGetVersionString(MAKE_VERSION(5,1,9), 0, 0, buf, sizeof(buf))
1272 << endl;
1273 exitHandler(NDBT_FAILED);
1274 }
1275
1276 if (version > NDB_VERSION)
1277 {
1278 err << "Restore program older than backup version. Not supported. "
1279 << "Use new restore program" << endl;
1280 exitHandler(NDBT_FAILED);
1281 }
1282
1283 debug << "Load content" << endl;
1284 int res = metaData.loadContent();
1285
1286 info << "Stop GCP of Backup: " << metaData.getStopGCP() << endl;
1287
1288 if (res == 0)
1289 {
1290 err << "Restore: Failed to load content" << endl;
1291 exitHandler(NDBT_FAILED);
1292 }
1293 debug << "Get no of Tables" << endl;
1294 if (metaData.getNoOfTables() == 0)
1295 {
1296 err << "The backup contains no tables" << endl;
1297 exitHandler(NDBT_FAILED);
1298 }
1299 debug << "Validate Footer" << endl;
1300
1301 if (!metaData.validateFooter())
1302 {
1303 err << "Restore: Failed to validate footer." << endl;
1304 exitHandler(NDBT_FAILED);
1305 }
1306 debug << "Init Backup objects" << endl;
1307 Uint32 i;
1308 for(i= 0; i < g_consumers.size(); i++)
1309 {
1310 if (!g_consumers[i]->init(g_tableCompabilityMask))
1311 {
1312 clearConsumers();
1313 err << "Failed to initialize consumers" << endl;
1314 exitHandler(NDBT_FAILED);
1315 }
1316
1317 }
1318
1319 /* report to clusterlog if applicable */
1320 for (i = 0; i < g_consumers.size(); i++)
1321 g_consumers[i]->report_started(ga_backupId, ga_nodeId);
1322
1323 debug << "Restore objects (tablespaces, ..)" << endl;
1324 for(i = 0; i<metaData.getNoOfObjects(); i++)
1325 {
1326 for(Uint32 j= 0; j < g_consumers.size(); j++)
1327 if (!g_consumers[j]->object(metaData.getObjType(i),
1328 metaData.getObjPtr(i)))
1329 {
1330 err << "Restore: Failed to restore table: ";
1331 err << metaData[i]->getTableName() << " ... Exiting " << endl;
1332 exitHandler(NDBT_FAILED);
1333 }
1334 if (check_progress())
1335 {
1336 info.setLevel(255);
1337 info << "Object create progress: "
1338 << i+1 << " objects out of "
1339 << metaData.getNoOfObjects() << endl;
1340 }
1341 }
1342
1343 Vector<OutputStream *> table_output(metaData.getNoOfTables());
1344 debug << "Restoring tables" << endl;
1345 for(i = 0; i<metaData.getNoOfTables(); i++)
1346 {
1347 const TableS *table= metaData[i];
1348 table_output.push_back(NULL);
1349 if (!checkDbAndTableName(table))
1350 continue;
1351 if (isSYSTAB_0(table))
1352 {
1353 table_output[i]= ndbout.m_out;
1354 }
1355 if (checkSysTable(table))
1356 {
1357 if (!tab_path || isBlobTable(table) || isIndex(table))
1358 {
1359 table_output[i]= ndbout.m_out;
1360 }
1361 else
1362 {
1363 FILE* res;
1364 char filename[FN_REFLEN], tmp_path[FN_REFLEN];
1365 const char *table_name;
1366 table_name= table->getTableName();
1367 while (*table_name != '/') table_name++;
1368 table_name++;
1369 while (*table_name != '/') table_name++;
1370 table_name++;
1371 convert_dirname(tmp_path, tab_path, NullS);
1372 res= my_fopen(fn_format(filename, table_name, tmp_path, ".txt", 4),
1373 opt_append ?
1374 O_WRONLY|O_APPEND|O_CREAT :
1375 O_WRONLY|O_TRUNC|O_CREAT,
1376 MYF(MY_WME));
1377 if (res == 0)
1378 {
1379 exitHandler(NDBT_FAILED);
1380 }
1381 FileOutputStream *f= new FileOutputStream(res);
1382 table_output[i]= f;
1383 }
1384 for(Uint32 j= 0; j < g_consumers.size(); j++)
1385 if (!g_consumers[j]->table(* table))
1386 {
1387 err << "Restore: Failed to restore table: `";
1388 err << table->getTableName() << "` ... Exiting " << endl;
1389 exitHandler(NDBT_FAILED);
1390 }
1391 } else {
1392 for(Uint32 j= 0; j < g_consumers.size(); j++)
1393 if (!g_consumers[j]->createSystable(* table))
1394 {
1395 err << "Restore: Failed to restore system table: ";
1396 err << table->getTableName() << " ... Exiting " << endl;
1397 exitHandler(NDBT_FAILED);
1398 }
1399 }
1400 if (check_progress())
1401 {
1402 info.setLevel(255);
1403 info << "Table create progress: "
1404 << i+1 << " tables out of "
1405 << metaData.getNoOfTables() << endl;
1406 }
1407 }
1408 debug << "Close tables" << endl;
1409 for(i= 0; i < g_consumers.size(); i++)
1410 if (!g_consumers[i]->endOfTables())
1411 {
1412 err << "Restore: Failed while closing tables" << endl;
1413 exitHandler(NDBT_FAILED);
1414 }
1415 /* report to clusterlog if applicable */
1416 for(i= 0; i < g_consumers.size(); i++)
1417 {
1418 g_consumers[i]->report_meta_data(ga_backupId, ga_nodeId);
1419 }
1420 debug << "Iterate over data" << endl;
1421 if (ga_restore || ga_print)
1422 {
1423 if(_restore_data || _print_data)
1424 {
1425 // Check table compatibility
1426 for (i=0; i < metaData.getNoOfTables(); i++){
1427 if (checkSysTable(metaData, i) &&
1428 checkDbAndTableName(metaData[i]))
1429 {
1430 for(Uint32 j= 0; j < g_consumers.size(); j++)
1431 {
1432 if (!g_consumers[j]->table_compatible_check(*metaData[i]))
1433 {
1434 err << "Restore: Failed to restore data, ";
1435 err << metaData[i]->getTableName() << " table structure incompatible with backup's ... Exiting " << endl;
1436 exitHandler(NDBT_FAILED);
1437 }
1438 }
1439 }
1440 }
1441 RestoreDataIterator dataIter(metaData, &free_data_callback);
1442
1443 // Read data file header
1444 if (!dataIter.readHeader())
1445 {
1446 err << "Failed to read header of data file. Exiting..." << endl;
1447 exitHandler(NDBT_FAILED);
1448 }
1449
1450 Uint32 fragmentId;
1451 while (dataIter.readFragmentHeader(res= 0, &fragmentId))
1452 {
1453 const TupleS* tuple;
1454 while ((tuple = dataIter.getNextTuple(res= 1)) != 0)
1455 {
1456 const TableS* table = tuple->getTable();
1457 OutputStream *output = table_output[table->getLocalId()];
1458 if (!output)
1459 continue;
1460 OutputStream *tmp = ndbout.m_out;
1461 ndbout.m_out = output;
1462 for(Uint32 j= 0; j < g_consumers.size(); j++)
1463 g_consumers[j]->tuple(* tuple, fragmentId);
1464 ndbout.m_out = tmp;
1465 if (check_progress())
1466 report_progress("Data file progress: ", dataIter);
1467 } // while (tuple != NULL);
1468
1469 if (res < 0)
1470 {
1471 err <<" Restore: An error occured while restoring data. Exiting...";
1472 err << endl;
1473 exitHandler(NDBT_FAILED);
1474 }
1475 if (!dataIter.validateFragmentFooter()) {
1476 err << "Restore: Error validating fragment footer. ";
1477 err << "Exiting..." << endl;
1478 exitHandler(NDBT_FAILED);
1479 }
1480 } // while (dataIter.readFragmentHeader(res))
1481
1482 if (res < 0)
1483 {
1484 err << "Restore: An error occured while restoring data. Exiting... "
1485 << "res= " << res << endl;
1486 exitHandler(NDBT_FAILED);
1487 }
1488
1489
1490 dataIter.validateFooter(); //not implemented
1491
1492 for (i= 0; i < g_consumers.size(); i++)
1493 g_consumers[i]->endOfTuples();
1494
1495 /* report to clusterlog if applicable */
1496 for(i= 0; i < g_consumers.size(); i++)
1497 {
1498 g_consumers[i]->report_data(ga_backupId, ga_nodeId);
1499 }
1500 }
1501
1502 if(_restore_data || _print_log)
1503 {
1504 RestoreLogIterator logIter(metaData);
1505 if (!logIter.readHeader())
1506 {
1507 err << "Failed to read header of data file. Exiting..." << endl;
1508 exitHandler(NDBT_FAILED);
1509 }
1510
1511 const LogEntry * logEntry = 0;
1512 while ((logEntry = logIter.getNextLogEntry(res= 0)) != 0)
1513 {
1514 const TableS* table = logEntry->m_table;
1515 OutputStream *output = table_output[table->getLocalId()];
1516 if (!output)
1517 continue;
1518 for(Uint32 j= 0; j < g_consumers.size(); j++)
1519 g_consumers[j]->logEntry(* logEntry);
1520
1521 if (check_progress())
1522 report_progress("Log file progress: ", logIter);
1523 }
1524 if (res < 0)
1525 {
1526 err << "Restore: An restoring the data log. Exiting... res="
1527 << res << endl;
1528 exitHandler(NDBT_FAILED);
1529 }
1530 logIter.validateFooter(); //not implemented
1531 for (i= 0; i < g_consumers.size(); i++)
1532 g_consumers[i]->endOfLogEntrys();
1533
1534 /* report to clusterlog if applicable */
1535 for(i= 0; i < g_consumers.size(); i++)
1536 {
1537 g_consumers[i]->report_log(ga_backupId, ga_nodeId);
1538 }
1539 }
1540
1541 if(_restore_data)
1542 {
1543 for(i = 0; i < metaData.getNoOfTables(); i++)
1544 {
1545 const TableS* table = metaData[i];
1546 check_data_truncations(table);
1547 OutputStream *output = table_output[table->getLocalId()];
1548 if (!output)
1549 continue;
1550 for(Uint32 j= 0; j < g_consumers.size(); j++)
1551 if (!g_consumers[j]->finalize_table(*table))
1552 {
1553 err << "Restore: Failed to finalize restore table: %s. ";
1554 err << "Exiting... " << metaData[i]->getTableName() << endl;
1555 exitHandler(NDBT_FAILED);
1556 }
1557 }
1558 }
1559 }
1560 if (ga_restore_epoch)
1561 {
1562 for (i= 0; i < g_consumers.size(); i++)
1563 if (!g_consumers[i]->update_apply_status(metaData))
1564 {
1565 err << "Restore: Failed to restore epoch" << endl;
1566 return -1;
1567 }
1568 }
1569
1570 unsigned j;
1571 for(j= 0; j < g_consumers.size(); j++)
1572 {
1573 if (g_consumers[j]->has_temp_error())
1574 {
1575 clearConsumers();
1576 ndbout_c("\nRestore successful, but encountered temporary error, "
1577 "please look at configuration.");
1578 }
1579 }
1580
1581 if (ga_rebuild_indexes)
1582 {
1583 debug << "Rebuilding indexes" << endl;
1584 for(i = 0; i<metaData.getNoOfTables(); i++)
1585 {
1586 const TableS *table= metaData[i];
1587 if (! (checkSysTable(table) && checkDbAndTableName(table)))
1588 continue;
1589 if (isBlobTable(table) || isIndex(table))
1590 continue;
1591 for(Uint32 j= 0; j < g_consumers.size(); j++)
1592 {
1593 if (!g_consumers[j]->rebuild_indexes(* table))
1594 return -1;
1595 }
1596 }
1597 }
1598
1599 /* report to clusterlog if applicable */
1600 for (i = 0; i < g_consumers.size(); i++)
1601 g_consumers[i]->report_completed(ga_backupId, ga_nodeId);
1602
1603 clearConsumers();
1604
1605 for(i = 0; i < metaData.getNoOfTables(); i++)
1606 {
1607 if (table_output[i] &&
1608 table_output[i] != ndbout.m_out)
1609 {
1610 my_fclose(((FileOutputStream *)table_output[i])->getFile(), MYF(MY_WME));
1611 delete table_output[i];
1612 table_output[i] = NULL;
1613 }
1614 }
1615
1616 if (opt_verbose)
1617 return NDBT_ProgramExit(NDBT_OK);
1618 else
1619 return 0;
1620 } // main
1621
1622 template class Vector<BackupConsumer*>;
1623 template class Vector<OutputStream*>;
1624 template class Vector<RestoreOption *>;
1625