1 /*
2 Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License, version 2.0, for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 #include <NDBT_ReturnCodes.h>
26 #include "consumer_restore.hpp"
27 #include <kernel/ndb_limits.h>
28 #include <my_sys.h>
29 #include <NdbSleep.h>
30 #include <NdbTick.h>
31 #include <Properties.hpp>
32 #include <NdbTypesUtil.hpp>
33
34 #include <ndb_internal.hpp>
35 #include <ndb_logevent.h>
36 #include "../src/ndbapi/NdbDictionaryImpl.hpp"
37
38 #define NDB_ANYVALUE_FOR_NOLOGGING 0x8000007f
39
40 extern FilteredNdbOut err;
41 extern FilteredNdbOut info;
42 extern FilteredNdbOut debug;
43
44 static void callback(int, NdbTransaction*, void*);
45 static Uint32 get_part_id(const NdbDictionary::Table *table,
46 Uint32 hash_value);
47
48 extern BaseString g_options;
49 extern unsigned int opt_no_binlog;
50 extern bool ga_skip_broken_objects;
51
52 extern Properties g_rewrite_databases;
53
54 bool BackupRestore::m_preserve_trailing_spaces = false;
55
56 // ----------------------------------------------------------------------
57 // conversion handlers
58 // ----------------------------------------------------------------------
59
60 void *
convert_bitset(const void * source,void * target,bool & truncated)61 BackupRestore::convert_bitset(const void *source,
62 void *target,
63 bool &truncated)
64 {
65 if (!source || !target)
66 return NULL;
67
68 // shortcuts
69 const unsigned char * const s = (const unsigned char *)source;
70 char_n_padding_struct * const t = (char_n_padding_struct *)target;
71
72 // write data
73 if (t->n_new >= t->n_old)
74 {
75 // clear all bits
76 memset(t->new_row, 0, t->n_new);
77
78 memcpy(t->new_row, s, t->n_old);
79 truncated = false;
80 } else {
81 // set all bits, for parity with replication's demotion semantics
82 memset(t->new_row, 0xFF, t->n_new);
83 truncated = true;
84 }
85
86 return t->new_row;
87 }
88
89 template< typename S, typename T >
90 void *
convert_array(const void * source,void * target,bool & truncated)91 BackupRestore::convert_array(const void * source,
92 void * target,
93 bool & truncated)
94 {
95 if (!source || !target)
96 return NULL;
97
98 // shortcuts (note that all S::... and T::... are compile-time expr)
99 const unsigned char * const s = (const unsigned char *)source;
100 char_n_padding_struct * const t = (char_n_padding_struct *)target;
101 const Uint32 s_prefix_length = S::lengthPrefixSize();
102 const Uint32 t_prefix_length = T::lengthPrefixSize();
103
104 // read and adjust length
105 Uint32 length = (S::isFixedSized() ? t->n_old : S::readLengthPrefix(s));
106 const Uint32 max_length = t->n_new - t_prefix_length;
107 if (S::isFixedSized() && !m_preserve_trailing_spaces) {
108 const char s_padding_char = (S::isBinary() ? 0x00 : ' ');
109 // ignore padding chars for data copying or truncation reporting
110 while (length > 0 && s[length - 1] == s_padding_char) {
111 length--;
112 }
113 }
114 if (length <= max_length) {
115 truncated = false;
116 } else {
117 length = max_length;
118 truncated = true;
119 }
120
121 // write length prefix
122 if (!T::isFixedSized()) {
123 T::writeLengthPrefix(t->new_row, length);
124 }
125
126 // write data
127 memcpy(t->new_row + t_prefix_length, s + s_prefix_length, length);
128
129 // write padding
130 if (T::isFixedSized()) {
131 const char t_padding_char = (T::isBinary() ? 0x00 : ' ');
132 const Uint32 l = max_length - length;
133 memset(t->new_row + t_prefix_length + length, t_padding_char, l);
134 }
135
136 return t->new_row;
137 }
138
139 template< typename S, typename T >
140 void *
convert_integral(const void * source,void * target,bool & truncated)141 BackupRestore::convert_integral(const void * source,
142 void * target,
143 bool & truncated)
144 {
145 if (!source || !target)
146 return NULL;
147
148 // read the source value
149 typename S::DomainT s;
150 S::load(&s, (char *)source);
151
152 // Note: important to correctly handle mixed signedness comparisons.
153 //
154 // The problem: A straight-forward approach to convert value 's' into
155 // type 'T' might be to check into which of these subranges 's' falls
156 // ... < T's lower bound <= ... <= T's upper bound < ...
157 // However, this approach is _incorrect_ when applied to generic code
158 // if (s < T::lowest()) ... else if (s > T::highest()) ... else ...
159 // since 'S' and 'T' may be types of different signedness.
160 //
161 // Under ansi (and even more K&R) C promotion rules, if 'T' is unsigned
162 // and if there's no larger signed type available, the value 's' gets
163 // promoted to unsigned; then, a negative value of 's' becomes (large)
164 // positive -- with a wrong comparison outcome.
165 //
166 // Furthermore, the code should not trigger compiler warnings for any
167 // selection of integral types 'S', 'T' ("mixed signedness comparison",
168 // "comparison of unsigned expression <0 / >=0 is always false/true").
169 //
170 // The correct approach: do lower bound comparisons on signed types and
171 // upper bound comparisons on unsigned types only; this requires casts.
172 // For the casts to be safe, compare the value against the zero literal
173 // if (s <= 0) { check as signed } else { check as unsigned }
174 // which is a valid + nontrivial test for signed and unsigned types.
175 //
176 // This implies that correct, generic conversion code must test into
177 // which of these _four_ subranges value 's' falls
178 // ... < T's lower bound <= ... <= 0 < ... <= T's upper bound < ...
179 // while handling 's' as signed/unsigned where less-equal/greater zero.
180 //
181 // Obviously, simplifications are possible if 'S' is unsigned or known
182 // to be a subset of 'T'. This can be accomplished by a few additional
183 // compile-time expression tests, which allow code optimization to
184 // issue fewer checks for certain specializations of types 'S' and 'T'.
185
186 // write the target value
187 typename T::DomainT t;
188 if (s <= 0) {
189
190 // check value against lower bound as _signed_, safe since all <= 0
191 assert(S::lowest() <= 0 && T::lowest() <= 0 && s <= 0);
192 const typename S::SignedT s_l_s = S::asSigned(S::lowest());
193 const typename T::SignedT t_l_s = T::asSigned(T::lowest());
194 const typename S::SignedT s_s = S::asSigned(s);
195 if ((s_l_s < t_l_s) // compile-time expr
196 && (s_s < t_l_s)) { // lower bound check
197 t = T::lowest();
198 truncated = true;
199 } else { // within both bounds
200 t = static_cast< typename T::DomainT >(s);
201 truncated = false;
202 }
203
204 } else { // (s > 0)
205
206 // check value against upper bound as _unsigned_, safe since all > 0
207 assert(S::highest() > 0 && T::highest() > 0 && s > 0);
208 const typename S::UnsignedT s_h_u = S::asUnsigned(S::highest());
209 const typename T::UnsignedT t_h_u = T::asUnsigned(T::highest());
210 const typename S::UnsignedT s_u = S::asUnsigned(s);
211 if ((s_h_u > t_h_u) // compile-time expr
212 && (s_u > t_h_u)) { // upper bound check
213 t = T::highest();
214 truncated = true;
215 } else { // within both bounds
216 t = static_cast< typename T::DomainT >(s);
217 truncated = false;
218 }
219
220 }
221 T::store((char *)target, &t);
222
223 return target;
224 }
225
226 // ----------------------------------------------------------------------
227 // conversion rules
228 // ----------------------------------------------------------------------
229
230 const PromotionRules
231 BackupRestore::m_allowed_promotion_attrs[] = {
232 // bitset promotions/demotions
233 {NDBCOL::Bit, NDBCOL::Bit, check_compat_sizes,
234 convert_bitset},
235
236 // char array promotions/demotions
237 {NDBCOL::Char, NDBCOL::Char, check_compat_sizes,
238 convert_array< Hchar, Hchar >},
239 {NDBCOL::Char, NDBCOL::Varchar, check_compat_sizes,
240 convert_array< Hchar, Hvarchar >},
241 {NDBCOL::Char, NDBCOL::Longvarchar, check_compat_sizes,
242 convert_array< Hchar, Hlongvarchar >},
243 {NDBCOL::Varchar, NDBCOL::Char, check_compat_sizes,
244 convert_array< Hvarchar, Hchar >},
245 {NDBCOL::Varchar, NDBCOL::Varchar, check_compat_sizes,
246 convert_array< Hvarchar, Hvarchar >},
247 {NDBCOL::Varchar, NDBCOL::Longvarchar, check_compat_sizes,
248 convert_array< Hvarchar, Hlongvarchar >},
249 {NDBCOL::Longvarchar, NDBCOL::Char, check_compat_sizes,
250 convert_array< Hlongvarchar, Hchar >},
251 {NDBCOL::Longvarchar, NDBCOL::Varchar, check_compat_sizes,
252 convert_array< Hlongvarchar, Hvarchar >},
253 {NDBCOL::Longvarchar, NDBCOL::Longvarchar, check_compat_sizes,
254 convert_array< Hlongvarchar, Hlongvarchar >},
255
256 // binary array promotions/demotions
257 {NDBCOL::Binary, NDBCOL::Binary, check_compat_sizes,
258 convert_array< Hbinary, Hbinary >},
259 {NDBCOL::Binary, NDBCOL::Varbinary, check_compat_sizes,
260 convert_array< Hbinary, Hvarbinary >},
261 {NDBCOL::Binary, NDBCOL::Longvarbinary, check_compat_sizes,
262 convert_array< Hbinary, Hlongvarbinary >},
263 {NDBCOL::Varbinary, NDBCOL::Binary, check_compat_sizes,
264 convert_array< Hvarbinary, Hbinary >},
265 {NDBCOL::Varbinary, NDBCOL::Varbinary, check_compat_sizes,
266 convert_array< Hvarbinary, Hvarbinary >},
267 {NDBCOL::Varbinary, NDBCOL::Longvarbinary, check_compat_sizes,
268 convert_array< Hvarbinary, Hlongvarbinary >},
269 {NDBCOL::Longvarbinary, NDBCOL::Binary, check_compat_sizes,
270 convert_array< Hlongvarbinary, Hbinary >},
271 {NDBCOL::Longvarbinary, NDBCOL::Varbinary, check_compat_sizes,
272 convert_array< Hlongvarbinary, Hvarbinary >},
273 {NDBCOL::Longvarbinary, NDBCOL::Longvarbinary, check_compat_sizes,
274 convert_array< Hlongvarbinary, Hlongvarbinary >},
275
276 // integral promotions
277 {NDBCOL::Tinyint, NDBCOL::Smallint, check_compat_promotion,
278 convert_integral< Hint8, Hint16>},
279 {NDBCOL::Tinyint, NDBCOL::Mediumint, check_compat_promotion,
280 convert_integral< Hint8, Hint24>},
281 {NDBCOL::Tinyint, NDBCOL::Int, check_compat_promotion,
282 convert_integral< Hint8, Hint32>},
283 {NDBCOL::Tinyint, NDBCOL::Bigint, check_compat_promotion,
284 convert_integral< Hint8, Hint64>},
285 {NDBCOL::Smallint, NDBCOL::Mediumint, check_compat_promotion,
286 convert_integral< Hint16, Hint24>},
287 {NDBCOL::Smallint, NDBCOL::Int, check_compat_promotion,
288 convert_integral< Hint16, Hint32>},
289 {NDBCOL::Smallint, NDBCOL::Bigint, check_compat_promotion,
290 convert_integral< Hint16, Hint64>},
291 {NDBCOL::Mediumint, NDBCOL::Int, check_compat_promotion,
292 convert_integral< Hint24, Hint32>},
293 {NDBCOL::Mediumint, NDBCOL::Bigint, check_compat_promotion,
294 convert_integral< Hint24, Hint64>},
295 {NDBCOL::Int, NDBCOL::Bigint, check_compat_promotion,
296 convert_integral< Hint32, Hint64>},
297 {NDBCOL::Tinyunsigned, NDBCOL::Smallunsigned, check_compat_promotion,
298 convert_integral< Huint8, Huint16>},
299 {NDBCOL::Tinyunsigned, NDBCOL::Mediumunsigned, check_compat_promotion,
300 convert_integral< Huint8, Huint24>},
301 {NDBCOL::Tinyunsigned, NDBCOL::Unsigned, check_compat_promotion,
302 convert_integral< Huint8, Huint32>},
303 {NDBCOL::Tinyunsigned, NDBCOL::Bigunsigned, check_compat_promotion,
304 convert_integral< Huint8, Huint64>},
305 {NDBCOL::Smallunsigned, NDBCOL::Mediumunsigned, check_compat_promotion,
306 convert_integral< Huint16, Huint24>},
307 {NDBCOL::Smallunsigned, NDBCOL::Unsigned, check_compat_promotion,
308 convert_integral< Huint16, Huint32>},
309 {NDBCOL::Smallunsigned, NDBCOL::Bigunsigned, check_compat_promotion,
310 convert_integral< Huint16, Huint64>},
311 {NDBCOL::Mediumunsigned, NDBCOL::Unsigned, check_compat_promotion,
312 convert_integral< Huint24, Huint32>},
313 {NDBCOL::Mediumunsigned, NDBCOL::Bigunsigned, check_compat_promotion,
314 convert_integral< Huint24, Huint64>},
315 {NDBCOL::Unsigned, NDBCOL::Bigunsigned, check_compat_promotion,
316 convert_integral< Huint32, Huint64>},
317
318 // integral demotions
319 {NDBCOL::Smallint, NDBCOL::Tinyint, check_compat_lossy,
320 convert_integral< Hint16, Hint8>},
321 {NDBCOL::Mediumint, NDBCOL::Tinyint, check_compat_lossy,
322 convert_integral< Hint24, Hint8>},
323 {NDBCOL::Mediumint, NDBCOL::Smallint, check_compat_lossy,
324 convert_integral< Hint24, Hint16>},
325 {NDBCOL::Int, NDBCOL::Tinyint, check_compat_lossy,
326 convert_integral< Hint32, Hint8>},
327 {NDBCOL::Int, NDBCOL::Smallint, check_compat_lossy,
328 convert_integral< Hint32, Hint16>},
329 {NDBCOL::Int, NDBCOL::Mediumint, check_compat_lossy,
330 convert_integral< Hint32, Hint24>},
331 {NDBCOL::Bigint, NDBCOL::Tinyint, check_compat_lossy,
332 convert_integral< Hint64, Hint8>},
333 {NDBCOL::Bigint, NDBCOL::Smallint, check_compat_lossy,
334 convert_integral< Hint64, Hint16>},
335 {NDBCOL::Bigint, NDBCOL::Mediumint, check_compat_lossy,
336 convert_integral< Hint64, Hint24>},
337 {NDBCOL::Bigint, NDBCOL::Int, check_compat_lossy,
338 convert_integral< Hint64, Hint32>},
339 {NDBCOL::Smallunsigned, NDBCOL::Tinyunsigned, check_compat_lossy,
340 convert_integral< Huint16, Huint8>},
341 {NDBCOL::Mediumunsigned, NDBCOL::Tinyunsigned, check_compat_lossy,
342 convert_integral< Huint24, Huint8>},
343 {NDBCOL::Mediumunsigned, NDBCOL::Smallunsigned, check_compat_lossy,
344 convert_integral< Huint24, Huint16>},
345 {NDBCOL::Unsigned, NDBCOL::Tinyunsigned, check_compat_lossy,
346 convert_integral< Huint32, Huint8>},
347 {NDBCOL::Unsigned, NDBCOL::Smallunsigned, check_compat_lossy,
348 convert_integral< Huint32, Huint16>},
349 {NDBCOL::Unsigned, NDBCOL::Mediumunsigned, check_compat_lossy,
350 convert_integral< Huint32, Huint24>},
351 {NDBCOL::Bigunsigned, NDBCOL::Tinyunsigned, check_compat_lossy,
352 convert_integral< Huint64, Huint8>},
353 {NDBCOL::Bigunsigned, NDBCOL::Smallunsigned, check_compat_lossy,
354 convert_integral< Huint64, Huint16>},
355 {NDBCOL::Bigunsigned, NDBCOL::Mediumunsigned, check_compat_lossy,
356 convert_integral< Huint64, Huint24>},
357 {NDBCOL::Bigunsigned, NDBCOL::Unsigned, check_compat_lossy,
358 convert_integral< Huint64, Huint32>},
359
360 // integral signedness conversions
361 {NDBCOL::Tinyint, NDBCOL::Tinyunsigned, check_compat_lossy,
362 convert_integral< Hint8, Huint8>},
363 {NDBCOL::Smallint, NDBCOL::Smallunsigned, check_compat_lossy,
364 convert_integral< Hint16, Huint16>},
365 {NDBCOL::Mediumint, NDBCOL::Mediumunsigned, check_compat_lossy,
366 convert_integral< Hint24, Huint24>},
367 {NDBCOL::Int, NDBCOL::Unsigned, check_compat_lossy,
368 convert_integral< Hint32, Huint32>},
369 {NDBCOL::Bigint, NDBCOL::Bigunsigned, check_compat_lossy,
370 convert_integral< Hint64, Huint64>},
371 {NDBCOL::Tinyunsigned, NDBCOL::Tinyint, check_compat_lossy,
372 convert_integral< Huint8, Hint8>},
373 {NDBCOL::Smallunsigned, NDBCOL::Smallint, check_compat_lossy,
374 convert_integral< Huint16, Hint16>},
375 {NDBCOL::Mediumunsigned, NDBCOL::Mediumint, check_compat_lossy,
376 convert_integral< Huint24, Hint24>},
377 {NDBCOL::Unsigned, NDBCOL::Int, check_compat_lossy,
378 convert_integral< Huint32, Hint32>},
379 {NDBCOL::Bigunsigned, NDBCOL::Bigint, check_compat_lossy,
380 convert_integral< Huint64, Hint64>},
381
382 // integral signedness+promotion conversions
383 {NDBCOL::Tinyint, NDBCOL::Smallunsigned, check_compat_lossy,
384 convert_integral< Hint8, Huint16>},
385 {NDBCOL::Tinyint, NDBCOL::Mediumunsigned, check_compat_lossy,
386 convert_integral< Hint8, Huint24>},
387 {NDBCOL::Tinyint, NDBCOL::Unsigned, check_compat_lossy,
388 convert_integral< Hint8, Huint32>},
389 {NDBCOL::Tinyint, NDBCOL::Bigunsigned, check_compat_lossy,
390 convert_integral< Hint8, Huint64>},
391 {NDBCOL::Smallint, NDBCOL::Mediumunsigned, check_compat_lossy,
392 convert_integral< Hint16, Huint24>},
393 {NDBCOL::Smallint, NDBCOL::Unsigned, check_compat_lossy,
394 convert_integral< Hint16, Huint32>},
395 {NDBCOL::Smallint, NDBCOL::Bigunsigned, check_compat_lossy,
396 convert_integral< Hint16, Huint64>},
397 {NDBCOL::Mediumint, NDBCOL::Unsigned, check_compat_lossy,
398 convert_integral< Hint24, Huint32>},
399 {NDBCOL::Mediumint, NDBCOL::Bigunsigned, check_compat_lossy,
400 convert_integral< Hint24, Huint64>},
401 {NDBCOL::Int, NDBCOL::Bigunsigned, check_compat_lossy,
402 convert_integral< Hint32, Huint64>},
403 {NDBCOL::Tinyunsigned, NDBCOL::Smallint, check_compat_lossy,
404 convert_integral< Huint8, Hint16>},
405 {NDBCOL::Tinyunsigned, NDBCOL::Mediumint, check_compat_lossy,
406 convert_integral< Huint8, Hint24>},
407 {NDBCOL::Tinyunsigned, NDBCOL::Int, check_compat_lossy,
408 convert_integral< Huint8, Hint32>},
409 {NDBCOL::Tinyunsigned, NDBCOL::Bigint, check_compat_lossy,
410 convert_integral< Huint8, Hint64>},
411 {NDBCOL::Smallunsigned, NDBCOL::Mediumint, check_compat_lossy,
412 convert_integral< Huint16, Hint24>},
413 {NDBCOL::Smallunsigned, NDBCOL::Int, check_compat_lossy,
414 convert_integral< Huint16, Hint32>},
415 {NDBCOL::Smallunsigned, NDBCOL::Bigint, check_compat_lossy,
416 convert_integral< Huint16, Hint64>},
417 {NDBCOL::Mediumunsigned, NDBCOL::Int, check_compat_lossy,
418 convert_integral< Huint24, Hint32>},
419 {NDBCOL::Mediumunsigned, NDBCOL::Bigint, check_compat_lossy,
420 convert_integral< Huint24, Hint64>},
421 {NDBCOL::Unsigned, NDBCOL::Bigint, check_compat_lossy,
422 convert_integral< Huint32, Hint64>},
423
424 // integral signedness+demotion conversions
425 {NDBCOL::Smallint, NDBCOL::Tinyunsigned, check_compat_lossy,
426 convert_integral< Hint16, Huint8>},
427 {NDBCOL::Mediumint, NDBCOL::Tinyunsigned, check_compat_lossy,
428 convert_integral< Hint24, Huint8>},
429 {NDBCOL::Mediumint, NDBCOL::Smallunsigned, check_compat_lossy,
430 convert_integral< Hint24, Huint16>},
431 {NDBCOL::Int, NDBCOL::Tinyunsigned, check_compat_lossy,
432 convert_integral< Hint32, Huint8>},
433 {NDBCOL::Int, NDBCOL::Smallunsigned, check_compat_lossy,
434 convert_integral< Hint32, Huint16>},
435 {NDBCOL::Int, NDBCOL::Mediumunsigned, check_compat_lossy,
436 convert_integral< Hint32, Huint24>},
437 {NDBCOL::Bigint, NDBCOL::Tinyunsigned, check_compat_lossy,
438 convert_integral< Hint64, Huint8>},
439 {NDBCOL::Bigint, NDBCOL::Smallunsigned, check_compat_lossy,
440 convert_integral< Hint64, Huint16>},
441 {NDBCOL::Bigint, NDBCOL::Mediumunsigned, check_compat_lossy,
442 convert_integral< Hint64, Huint24>},
443 {NDBCOL::Bigint, NDBCOL::Unsigned, check_compat_lossy,
444 convert_integral< Hint64, Huint32>},
445 {NDBCOL::Smallunsigned, NDBCOL::Tinyint, check_compat_lossy,
446 convert_integral< Huint16, Hint8>},
447 {NDBCOL::Mediumunsigned, NDBCOL::Tinyint, check_compat_lossy,
448 convert_integral< Huint24, Hint8>},
449 {NDBCOL::Mediumunsigned, NDBCOL::Smallint, check_compat_lossy,
450 convert_integral< Huint24, Hint16>},
451 {NDBCOL::Unsigned, NDBCOL::Tinyint, check_compat_lossy,
452 convert_integral< Huint32, Hint8>},
453 {NDBCOL::Unsigned, NDBCOL::Smallint, check_compat_lossy,
454 convert_integral< Huint32, Hint16>},
455 {NDBCOL::Unsigned, NDBCOL::Mediumint, check_compat_lossy,
456 convert_integral< Huint32, Hint24>},
457 {NDBCOL::Bigunsigned, NDBCOL::Tinyint, check_compat_lossy,
458 convert_integral< Huint64, Hint8>},
459 {NDBCOL::Bigunsigned, NDBCOL::Smallint, check_compat_lossy,
460 convert_integral< Huint64, Hint16>},
461 {NDBCOL::Bigunsigned, NDBCOL::Mediumint, check_compat_lossy,
462 convert_integral< Huint64, Hint24>},
463 {NDBCOL::Bigunsigned, NDBCOL::Int, check_compat_lossy,
464 convert_integral< Huint64, Hint32>},
465
466 {NDBCOL::Undefined, NDBCOL::Undefined, NULL, NULL}
467 };
468
469 bool
init(Uint32 tableChangesMask)470 BackupRestore::init(Uint32 tableChangesMask)
471 {
472 release();
473
474 if (!m_restore && !m_restore_meta && !m_restore_epoch &&
475 !m_rebuild_indexes && !m_disable_indexes)
476 return true;
477
478 m_tableChangesMask = tableChangesMask;
479 m_cluster_connection = new Ndb_cluster_connection(m_ndb_connectstring,
480 m_ndb_nodeid);
481 if (m_cluster_connection == NULL)
482 {
483 err << "Failed to create cluster connection!!" << endl;
484 return false;
485 }
486 m_cluster_connection->set_name(g_options.c_str());
487 if(m_cluster_connection->connect(12, 5, 1) != 0)
488 {
489 return false;
490 }
491
492 m_ndb = new Ndb(m_cluster_connection);
493
494 if (m_ndb == NULL)
495 return false;
496
497 m_ndb->init(1024);
498 if (m_ndb->waitUntilReady(30) != 0)
499 {
500 err << "Failed to connect to ndb!!" << endl;
501 return false;
502 }
503 info << "Connected to ndb!!" << endl;
504
505 m_callback = new restore_callback_t[m_parallelism];
506
507 if (m_callback == 0)
508 {
509 err << "Failed to allocate callback structs" << endl;
510 return false;
511 }
512
513 m_free_callback= m_callback;
514 for (Uint32 i= 0; i < m_parallelism; i++) {
515 m_callback[i].restore= this;
516 m_callback[i].connection= 0;
517 if (i > 0)
518 m_callback[i-1].next= &(m_callback[i]);
519 }
520 m_callback[m_parallelism-1].next = 0;
521
522 return true;
523 }
524
release()525 void BackupRestore::release()
526 {
527 if (m_ndb)
528 {
529 delete m_ndb;
530 m_ndb= 0;
531 }
532
533 if (m_callback)
534 {
535 delete [] m_callback;
536 m_callback= 0;
537 }
538
539 if (m_cluster_connection)
540 {
541 delete m_cluster_connection;
542 m_cluster_connection= 0;
543 }
544 }
545
~BackupRestore()546 BackupRestore::~BackupRestore()
547 {
548 release();
549 }
550
551 static
552 int
match_blob(const char * name)553 match_blob(const char * name){
554 int cnt, id1, id2;
555 char buf[256];
556 if((cnt = sscanf(name, "%[^/]/%[^/]/NDB$BLOB_%d_%d", buf, buf, &id1, &id2)) == 4){
557 return id1;
558 }
559
560 return -1;
561 }
562
563 /**
564 * Extracts the database, schema, and table name from an internal table name;
565 * prints an error message and returns false in case of a format violation.
566 */
567 static
568 bool
dissect_table_name(const char * qualified_table_name,BaseString & db_name,BaseString & schema_name,BaseString & table_name)569 dissect_table_name(const char * qualified_table_name,
570 BaseString & db_name,
571 BaseString & schema_name,
572 BaseString & table_name) {
573 Vector<BaseString> split;
574 BaseString tmp(qualified_table_name);
575 if (tmp.split(split, "/") != 3) {
576 err << "Invalid table name format `" << qualified_table_name
577 << "`" << endl;
578 return false;
579 }
580 db_name = split[0];
581 schema_name = split[1];
582 table_name = split[2];
583 return true;
584 }
585
586 /**
587 * Assigns the new name for a database, if and only if to be rewritten.
588 */
589 static
590 void
check_rewrite_database(BaseString & db_name)591 check_rewrite_database(BaseString & db_name) {
592 const char * new_db_name;
593 if (g_rewrite_databases.get(db_name.c_str(), &new_db_name))
594 db_name.assign(new_db_name);
595 }
596
597 const NdbDictionary::Table*
get_table(const NdbDictionary::Table * tab)598 BackupRestore::get_table(const NdbDictionary::Table* tab){
599 if(m_cache.m_old_table == tab)
600 return m_cache.m_new_table;
601 m_cache.m_old_table = tab;
602
603 int cnt, id1, id2;
604 char db[256], schema[256];
605 if (strcmp(tab->getName(), "SYSTAB_0") == 0 ||
606 strcmp(tab->getName(), "sys/def/SYSTAB_0") == 0) {
607 /*
608 Restore SYSTAB_0 to itself
609 */
610 m_cache.m_new_table = tab;
611 }
612 else if((cnt = sscanf(tab->getName(), "%[^/]/%[^/]/NDB$BLOB_%d_%d",
613 db, schema, &id1, &id2)) == 4){
614 m_ndb->setDatabaseName(db);
615 m_ndb->setSchemaName(schema);
616
617 BaseString::snprintf(db, sizeof(db), "NDB$BLOB_%d_%d",
618 m_new_tables[id1]->getTableId(), id2);
619
620 m_cache.m_new_table = m_ndb->getDictionary()->getTable(db);
621
622 } else {
623 m_cache.m_new_table = m_new_tables[tab->getTableId()];
624 }
625 assert(m_cache.m_new_table);
626 return m_cache.m_new_table;
627 }
628
629 bool
finalize_table(const TableS & table)630 BackupRestore::finalize_table(const TableS & table){
631 bool ret= true;
632 if (!m_restore && !m_restore_meta)
633 return ret;
634 if (!table.have_auto_inc())
635 return ret;
636
637 Uint64 max_val= table.get_max_auto_val();
638 do
639 {
640 Uint64 auto_val = ~(Uint64)0;
641 int r= m_ndb->readAutoIncrementValue(get_table(table.m_dictTable), auto_val);
642 if (r == -1 && m_ndb->getNdbError().status == NdbError::TemporaryError)
643 {
644 NdbSleep_MilliSleep(50);
645 continue; // retry
646 }
647 else if (r == -1 && m_ndb->getNdbError().code != 626)
648 {
649 ret= false;
650 }
651 else if ((r == -1 && m_ndb->getNdbError().code == 626) ||
652 max_val+1 > auto_val || auto_val == ~(Uint64)0)
653 {
654 r= m_ndb->setAutoIncrementValue(get_table(table.m_dictTable),
655 max_val+1, false);
656 if (r == -1 &&
657 m_ndb->getNdbError().status == NdbError::TemporaryError)
658 {
659 NdbSleep_MilliSleep(50);
660 continue; // retry
661 }
662 ret = (r == 0);
663 }
664 return (ret);
665 } while (1);
666 }
667
668 bool
rebuild_indexes(const TableS & table)669 BackupRestore::rebuild_indexes(const TableS& table)
670 {
671 const char *tablename = table.getTableName();
672
673 const NdbDictionary::Table * tab = get_table(table.m_dictTable);
674 Uint32 id = tab->getObjectId();
675 if (m_index_per_table.size() <= id)
676 return true;
677
678 BaseString db_name, schema_name, table_name;
679 if (!dissect_table_name(tablename, db_name, schema_name, table_name)) {
680 return false;
681 }
682 check_rewrite_database(db_name);
683
684 m_ndb->setDatabaseName(db_name.c_str());
685 m_ndb->setSchemaName(schema_name.c_str());
686 NdbDictionary::Dictionary* dict = m_ndb->getDictionary();
687
688 Vector<NdbDictionary::Index*> & indexes = m_index_per_table[id];
689 for(size_t i = 0; i<indexes.size(); i++)
690 {
691 const NdbDictionary::Index * const idx = indexes[i];
692 const char * const idx_name = idx->getName();
693 const char * const tab_name = idx->getTable();
694 Uint64 start = NdbTick_CurrentMillisecond();
695 info << "Rebuilding index `" << idx_name << "` on table `"
696 << tab_name << "` ..." << flush;
697 if ((dict->getIndex(idx_name, tab_name) == NULL)
698 && (dict->createIndex(* idx, 1) != 0))
699 {
700 info << "FAIL!" << endl;
701 err << "Rebuilding index `" << idx_name << "` on table `"
702 << tab_name <<"` failed: ";
703 err << dict->getNdbError() << endl;
704
705 return false;
706 }
707 Uint64 stop = NdbTick_CurrentMillisecond();
708 info << "OK (" << ((stop - start)/1000) << "s)" <<endl;
709 }
710
711 return true;
712 }
713
714 #ifdef NOT_USED
default_nodegroups(NdbDictionary::Table * table)715 static bool default_nodegroups(NdbDictionary::Table *table)
716 {
717 Uint16 *node_groups = (Uint16*)table->getFragmentData();
718 Uint32 no_parts = table->getFragmentDataLen() >> 1;
719 Uint32 i;
720
721 if (node_groups[0] != 0)
722 return false;
723 for (i = 1; i < no_parts; i++)
724 {
725 if (node_groups[i] != NDB_UNDEF_NODEGROUP)
726 return false;
727 }
728 return true;
729 }
730 #endif
731
732
get_no_fragments(Uint64 max_rows,Uint32 no_nodes)733 static Uint32 get_no_fragments(Uint64 max_rows, Uint32 no_nodes)
734 {
735 Uint32 i = 0;
736 Uint32 acc_row_size = 27;
737 Uint32 acc_fragment_size = 512*1024*1024;
738 Uint32 no_parts= Uint32((max_rows*acc_row_size)/acc_fragment_size + 1);
739 Uint32 reported_parts = no_nodes;
740 while (reported_parts < no_parts && ++i < 4 &&
741 (reported_parts + no_parts) < MAX_NDB_PARTITIONS)
742 reported_parts+= no_nodes;
743 if (reported_parts < no_parts)
744 {
745 err << "Table will be restored but will not be able to handle the maximum";
746 err << " amount of rows as requested" << endl;
747 }
748 return reported_parts;
749 }
750
751
set_default_nodegroups(NdbDictionary::Table * table)752 static void set_default_nodegroups(NdbDictionary::Table *table)
753 {
754 Uint32 no_parts = table->getFragmentCount();
755 Uint32 node_group[MAX_NDB_PARTITIONS];
756 Uint32 i;
757
758 node_group[0] = 0;
759 for (i = 1; i < no_parts; i++)
760 {
761 node_group[i] = NDB_UNDEF_NODEGROUP;
762 }
763 table->setFragmentData(node_group, no_parts);
764 }
765
map_ng(Uint32 ng)766 Uint32 BackupRestore::map_ng(Uint32 ng)
767 {
768 NODE_GROUP_MAP *ng_map = m_nodegroup_map;
769
770 if (ng == NDB_UNDEF_NODEGROUP ||
771 ng_map[ng].map_array[0] == NDB_UNDEF_NODEGROUP)
772 {
773 return ng;
774 }
775 else
776 {
777 Uint32 new_ng;
778 Uint32 curr_inx = ng_map[ng].curr_index;
779 Uint32 new_curr_inx = curr_inx + 1;
780
781 assert(ng < MAX_NDB_PARTITIONS);
782 assert(curr_inx < MAX_MAPS_PER_NODE_GROUP);
783 assert(new_curr_inx < MAX_MAPS_PER_NODE_GROUP);
784
785 if (new_curr_inx >= MAX_MAPS_PER_NODE_GROUP)
786 new_curr_inx = 0;
787 else if (ng_map[ng].map_array[new_curr_inx] == NDB_UNDEF_NODEGROUP)
788 new_curr_inx = 0;
789 new_ng = ng_map[ng].map_array[curr_inx];
790 ng_map[ng].curr_index = new_curr_inx;
791 return new_ng;
792 }
793 }
794
795
map_nodegroups(Uint32 * ng_array,Uint32 no_parts)796 bool BackupRestore::map_nodegroups(Uint32 *ng_array, Uint32 no_parts)
797 {
798 Uint32 i;
799 bool mapped = FALSE;
800 DBUG_ENTER("map_nodegroups");
801
802 assert(no_parts < MAX_NDB_PARTITIONS);
803 for (i = 0; i < no_parts; i++)
804 {
805 Uint32 ng;
806 ng = map_ng(ng_array[i]);
807 if (ng != ng_array[i])
808 mapped = TRUE;
809 ng_array[i] = ng;
810 }
811 DBUG_RETURN(mapped);
812 }
813
814
copy_byte(const char ** data,char ** new_data,uint * len)815 static void copy_byte(const char **data, char **new_data, uint *len)
816 {
817 **new_data = **data;
818 (*data)++;
819 (*new_data)++;
820 (*len)++;
821 }
822
823
search_replace(char * search_str,char ** new_data,const char ** data,const char * end_data,uint * new_data_len)824 bool BackupRestore::search_replace(char *search_str, char **new_data,
825 const char **data, const char *end_data,
826 uint *new_data_len)
827 {
828 uint search_str_len = strlen(search_str);
829 uint inx = 0;
830 bool in_delimiters = FALSE;
831 bool escape_char = FALSE;
832 char start_delimiter = 0;
833 DBUG_ENTER("search_replace");
834
835 do
836 {
837 char c = **data;
838 copy_byte(data, new_data, new_data_len);
839 if (escape_char)
840 {
841 escape_char = FALSE;
842 }
843 else if (in_delimiters)
844 {
845 if (c == start_delimiter)
846 in_delimiters = FALSE;
847 }
848 else if (c == '\'' || c == '\"')
849 {
850 in_delimiters = TRUE;
851 start_delimiter = c;
852 }
853 else if (c == '\\')
854 {
855 escape_char = TRUE;
856 }
857 else if (c == search_str[inx])
858 {
859 inx++;
860 if (inx == search_str_len)
861 {
862 bool found = FALSE;
863 uint number = 0;
864 while (*data != end_data)
865 {
866 if (isdigit(**data))
867 {
868 found = TRUE;
869 number = (10 * number) + (**data);
870 if (number > MAX_NDB_NODES)
871 break;
872 }
873 else if (found)
874 {
875 /*
876 After long and tedious preparations we have actually found
877 a node group identifier to convert. We'll use the mapping
878 table created for node groups and then insert the new number
879 instead of the old number.
880 */
881 uint temp = map_ng(number);
882 int no_digits = 0;
883 char digits[10];
884 while (temp != 0)
885 {
886 digits[no_digits] = temp % 10;
887 no_digits++;
888 temp/=10;
889 }
890 for (no_digits--; no_digits >= 0; no_digits--)
891 {
892 **new_data = digits[no_digits];
893 *new_data_len+=1;
894 }
895 DBUG_RETURN(FALSE);
896 }
897 else
898 break;
899 (*data)++;
900 }
901 DBUG_RETURN(TRUE);
902 }
903 }
904 else
905 inx = 0;
906 } while (*data < end_data);
907 DBUG_RETURN(FALSE);
908 }
909
map_in_frm(char * new_data,const char * data,uint data_len,uint * new_data_len)910 bool BackupRestore::map_in_frm(char *new_data, const char *data,
911 uint data_len, uint *new_data_len)
912 {
913 const char *end_data= data + data_len;
914 const char *end_part_data;
915 const char *part_data;
916 char *extra_ptr;
917 uint start_key_definition_len = uint2korr(data + 6);
918 uint key_definition_len = uint4korr(data + 47);
919 uint part_info_len;
920 DBUG_ENTER("map_in_frm");
921
922 if (data_len < 4096) goto error;
923 extra_ptr = (char*)data + start_key_definition_len + key_definition_len;
924 if ((int)data_len < ((extra_ptr - data) + 2)) goto error;
925 extra_ptr = extra_ptr + 2 + uint2korr(extra_ptr);
926 if ((int)data_len < ((extra_ptr - data) + 2)) goto error;
927 extra_ptr = extra_ptr + 2 + uint2korr(extra_ptr);
928 if ((int)data_len < ((extra_ptr - data) + 4)) goto error;
929 part_info_len = uint4korr(extra_ptr);
930 part_data = extra_ptr + 4;
931 if ((int)data_len < ((part_data + part_info_len) - data)) goto error;
932
933 do
934 {
935 copy_byte(&data, &new_data, new_data_len);
936 } while (data < part_data);
937 end_part_data = part_data + part_info_len;
938 do
939 {
940 if (search_replace((char*)" NODEGROUP = ", &new_data, &data,
941 end_part_data, new_data_len))
942 goto error;
943 } while (data != end_part_data);
944 do
945 {
946 copy_byte(&data, &new_data, new_data_len);
947 } while (data < end_data);
948 DBUG_RETURN(FALSE);
949 error:
950 DBUG_RETURN(TRUE);
951 }
952
953
translate_frm(NdbDictionary::Table * table)954 bool BackupRestore::translate_frm(NdbDictionary::Table *table)
955 {
956 uchar *pack_data, *data, *new_pack_data;
957 char *new_data;
958 uint new_data_len;
959 size_t data_len, new_pack_len;
960 uint no_parts, extra_growth;
961 DBUG_ENTER("translate_frm");
962
963 pack_data = (uchar*) table->getFrmData();
964 no_parts = table->getFragmentCount();
965 /*
966 Add max 4 characters per partition to handle worst case
967 of mapping from single digit to 5-digit number.
968 Fairly future-proof, ok up to 99999 node groups.
969 */
970 extra_growth = no_parts * 4;
971 if (unpackfrm(&data, &data_len, pack_data))
972 {
973 DBUG_RETURN(TRUE);
974 }
975 if ((new_data = (char*) malloc(data_len + extra_growth)))
976 {
977 DBUG_RETURN(TRUE);
978 }
979 if (map_in_frm(new_data, (const char*)data, data_len, &new_data_len))
980 {
981 free(new_data);
982 DBUG_RETURN(TRUE);
983 }
984 if (packfrm((uchar*) new_data, new_data_len,
985 &new_pack_data, &new_pack_len))
986 {
987 free(new_data);
988 DBUG_RETURN(TRUE);
989 }
990 table->setFrm(new_pack_data, (Uint32)new_pack_len);
991 DBUG_RETURN(FALSE);
992 }
993
994 #include <signaldata/DictTabInfo.hpp>
995
996 bool
object(Uint32 type,const void * ptr)997 BackupRestore::object(Uint32 type, const void * ptr)
998 {
999 if (!m_restore_meta)
1000 return true;
1001
1002 NdbDictionary::Dictionary* dict = m_ndb->getDictionary();
1003 switch(type){
1004 case DictTabInfo::Tablespace:
1005 {
1006 NdbDictionary::Tablespace old(*(NdbDictionary::Tablespace*)ptr);
1007
1008 Uint32 id = old.getObjectId();
1009
1010 if (!m_no_restore_disk)
1011 {
1012 NdbDictionary::LogfileGroup * lg = m_logfilegroups[old.getDefaultLogfileGroupId()];
1013 old.setDefaultLogfileGroup(* lg);
1014 info << "Creating tablespace: " << old.getName() << "..." << flush;
1015 int ret = dict->createTablespace(old);
1016 if (ret)
1017 {
1018 NdbError errobj= dict->getNdbError();
1019 info << "FAILED" << endl;
1020 err << "Create tablespace failed: " << old.getName() << ": " << errobj << endl;
1021 return false;
1022 }
1023 info << "done" << endl;
1024 }
1025
1026 NdbDictionary::Tablespace curr = dict->getTablespace(old.getName());
1027 NdbError errobj = dict->getNdbError();
1028 if ((int) errobj.classification == (int) ndberror_cl_none)
1029 {
1030 NdbDictionary::Tablespace* currptr = new NdbDictionary::Tablespace(curr);
1031 NdbDictionary::Tablespace * null = 0;
1032 m_tablespaces.set(currptr, id, null);
1033 debug << "Retreived tablespace: " << currptr->getName()
1034 << " oldid: " << id << " newid: " << currptr->getObjectId()
1035 << " " << (void*)currptr << endl;
1036 m_n_tablespace++;
1037 return true;
1038 }
1039
1040 err << "Failed to retrieve tablespace \"" << old.getName() << "\": "
1041 << errobj << endl;
1042
1043 return false;
1044 break;
1045 }
1046 case DictTabInfo::LogfileGroup:
1047 {
1048 NdbDictionary::LogfileGroup old(*(NdbDictionary::LogfileGroup*)ptr);
1049
1050 Uint32 id = old.getObjectId();
1051
1052 if (!m_no_restore_disk)
1053 {
1054 info << "Creating logfile group: " << old.getName() << "..." << flush;
1055 int ret = dict->createLogfileGroup(old);
1056 if (ret)
1057 {
1058 NdbError errobj= dict->getNdbError();
1059 info << "FAILED" << endl;
1060 err << "Create logfile group failed: " << old.getName() << ": " << errobj << endl;
1061 return false;
1062 }
1063 info << "done" << endl;
1064 }
1065
1066 NdbDictionary::LogfileGroup curr = dict->getLogfileGroup(old.getName());
1067 NdbError errobj = dict->getNdbError();
1068 if ((int) errobj.classification == (int) ndberror_cl_none)
1069 {
1070 NdbDictionary::LogfileGroup* currptr =
1071 new NdbDictionary::LogfileGroup(curr);
1072 NdbDictionary::LogfileGroup * null = 0;
1073 m_logfilegroups.set(currptr, id, null);
1074 debug << "Retreived logfile group: " << currptr->getName()
1075 << " oldid: " << id << " newid: " << currptr->getObjectId()
1076 << " " << (void*)currptr << endl;
1077 m_n_logfilegroup++;
1078 return true;
1079 }
1080
1081 err << "Failed to retrieve logfile group \"" << old.getName() << "\": "
1082 << errobj << endl;
1083
1084 return false;
1085 break;
1086 }
1087 case DictTabInfo::Datafile:
1088 {
1089 if (!m_no_restore_disk)
1090 {
1091 NdbDictionary::Datafile old(*(NdbDictionary::Datafile*)ptr);
1092 NdbDictionary::ObjectId objid;
1093 old.getTablespaceId(&objid);
1094 NdbDictionary::Tablespace * ts = m_tablespaces[objid.getObjectId()];
1095 debug << "Connecting datafile " << old.getPath()
1096 << " to tablespace: oldid: " << objid.getObjectId()
1097 << " newid: " << ts->getObjectId() << endl;
1098 old.setTablespace(* ts);
1099 info << "Creating datafile \"" << old.getPath() << "\"..." << flush;
1100 if (dict->createDatafile(old))
1101 {
1102 NdbError errobj= dict->getNdbError();
1103 info << "FAILED" << endl;
1104 err << "Create datafile failed: " << old.getPath() << ": " << errobj << endl;
1105 return false;
1106 }
1107 info << "done" << endl;
1108 m_n_datafile++;
1109 }
1110 return true;
1111 break;
1112 }
1113 case DictTabInfo::Undofile:
1114 {
1115 if (!m_no_restore_disk)
1116 {
1117 NdbDictionary::Undofile old(*(NdbDictionary::Undofile*)ptr);
1118 NdbDictionary::ObjectId objid;
1119 old.getLogfileGroupId(&objid);
1120 NdbDictionary::LogfileGroup * lg = m_logfilegroups[objid.getObjectId()];
1121 debug << "Connecting undofile " << old.getPath()
1122 << " to logfile group: oldid: " << objid.getObjectId()
1123 << " newid: " << lg->getObjectId()
1124 << " " << (void*)lg << endl;
1125 old.setLogfileGroup(* lg);
1126 info << "Creating undofile \"" << old.getPath() << "\"..." << flush;
1127 if (dict->createUndofile(old))
1128 {
1129 NdbError errobj= dict->getNdbError();
1130 info << "FAILED" << endl;
1131 err << "Create undofile failed: " << old.getPath() << ": " << errobj << endl;
1132 return false;
1133 }
1134 info << "done" << endl;
1135 m_n_undofile++;
1136 }
1137 return true;
1138 break;
1139 }
1140 case DictTabInfo::HashMap:
1141 {
1142 NdbDictionary::HashMap old(*(NdbDictionary::HashMap*)ptr);
1143
1144 Uint32 id = old.getObjectId();
1145
1146 if (m_restore_meta)
1147 {
1148 int ret = dict->createHashMap(old);
1149 if (ret == 0)
1150 {
1151 info << "Created hashmap: " << old.getName() << endl;
1152 }
1153 else
1154 {
1155 NdbError errobj = dict->getNdbError();
1156 // We ignore schema already exists, this is fine
1157 if (errobj.code != 721)
1158 {
1159 err << "Could not create hashmap \"" << old.getName() << "\": "
1160 << errobj << endl;
1161 return false;
1162 }
1163 }
1164 }
1165
1166 NdbDictionary::HashMap curr;
1167 if (dict->getHashMap(curr, old.getName()) == 0)
1168 {
1169 NdbDictionary::HashMap* currptr =
1170 new NdbDictionary::HashMap(curr);
1171 NdbDictionary::HashMap * null = 0;
1172 m_hashmaps.set(currptr, id, null);
1173 debug << "Retreived hashmap: " << currptr->getName()
1174 << " oldid: " << id << " newid: " << currptr->getObjectId()
1175 << " " << (void*)currptr << endl;
1176 return true;
1177 }
1178
1179 NdbError errobj = dict->getNdbError();
1180 err << "Failed to retrieve hashmap \"" << old.getName() << "\": "
1181 << errobj << endl;
1182
1183 return false;
1184 }
1185 default:
1186 {
1187 err << "Unknown object type: " << type << endl;
1188 break;
1189 }
1190 }
1191 return true;
1192 }
1193
1194 bool
has_temp_error()1195 BackupRestore::has_temp_error(){
1196 return m_temp_error;
1197 }
1198
1199 bool
update_apply_status(const RestoreMetaData & metaData)1200 BackupRestore::update_apply_status(const RestoreMetaData &metaData)
1201 {
1202 if (!m_restore_epoch)
1203 return true;
1204
1205 bool result= false;
1206 unsigned apply_table_format= 0;
1207
1208 m_ndb->setDatabaseName(NDB_REP_DB);
1209 m_ndb->setSchemaName("def");
1210
1211 NdbDictionary::Dictionary *dict= m_ndb->getDictionary();
1212 const NdbDictionary::Table *ndbtab= dict->getTable(NDB_APPLY_TABLE);
1213 if (!ndbtab)
1214 {
1215 err << NDB_APPLY_TABLE << ": "
1216 << dict->getNdbError() << endl;
1217 return false;
1218 }
1219 if (ndbtab->getColumn(0)->getType() == NdbDictionary::Column::Unsigned &&
1220 ndbtab->getColumn(1)->getType() == NdbDictionary::Column::Bigunsigned)
1221 {
1222 if (ndbtab->getNoOfColumns() == 2)
1223 {
1224 apply_table_format= 1;
1225 }
1226 else if
1227 (ndbtab->getColumn(2)->getType() == NdbDictionary::Column::Varchar &&
1228 ndbtab->getColumn(3)->getType() == NdbDictionary::Column::Bigunsigned &&
1229 ndbtab->getColumn(4)->getType() == NdbDictionary::Column::Bigunsigned)
1230 {
1231 apply_table_format= 2;
1232 }
1233 }
1234 if (apply_table_format == 0)
1235 {
1236 err << NDB_APPLY_TABLE << " has wrong format\n";
1237 return false;
1238 }
1239
1240 Uint32 server_id= 0;
1241 Uint64 epoch= Uint64(metaData.getStopGCP());
1242 Uint32 version= metaData.getNdbVersion();
1243
1244 /**
1245 * Bug#XXX, stopGCP is not really stop GCP, but stopGCP - 1
1246 */
1247 epoch += 1;
1248
1249 if (version >= NDBD_MICRO_GCP_63 ||
1250 (version >= NDBD_MICRO_GCP_62 && getMinor(version) == 2))
1251 {
1252 epoch<<= 32; // Only gci_hi is saved...
1253
1254 /**
1255 * Backup contains all epochs with those top bits,
1256 * so we indicate that with max setting
1257 */
1258 epoch += (Uint64(1) << 32) - 1;
1259 }
1260
1261 Uint64 zero= 0;
1262 char empty_string[1];
1263 empty_string[0]= 0;
1264 NdbTransaction * trans= m_ndb->startTransaction();
1265 if (!trans)
1266 {
1267 err << NDB_APPLY_TABLE << ": "
1268 << m_ndb->getNdbError() << endl;
1269 return false;
1270 }
1271 NdbOperation * op= trans->getNdbOperation(ndbtab);
1272 if (!op)
1273 {
1274 err << NDB_APPLY_TABLE << ": "
1275 << trans->getNdbError() << endl;
1276 goto err;
1277 }
1278 if (op->writeTuple() ||
1279 op->equal(0u, (const char *)&server_id, sizeof(server_id)) ||
1280 op->setValue(1u, (const char *)&epoch, sizeof(epoch)))
1281 {
1282 err << NDB_APPLY_TABLE << ": "
1283 << op->getNdbError() << endl;
1284 goto err;
1285 }
1286 if ((apply_table_format == 2) &&
1287 (op->setValue(2u, (const char *)&empty_string, 1) ||
1288 op->setValue(3u, (const char *)&zero, sizeof(zero)) ||
1289 op->setValue(4u, (const char *)&zero, sizeof(zero))))
1290 {
1291 err << NDB_APPLY_TABLE << ": "
1292 << op->getNdbError() << endl;
1293 goto err;
1294 }
1295 if (trans->execute(NdbTransaction::Commit))
1296 {
1297 err << NDB_APPLY_TABLE << ": "
1298 << trans->getNdbError() << endl;
1299 goto err;
1300 }
1301 result= true;
1302 err:
1303 m_ndb->closeTransaction(trans);
1304 return result;
1305 }
1306
1307 bool
report_started(unsigned backup_id,unsigned node_id)1308 BackupRestore::report_started(unsigned backup_id, unsigned node_id)
1309 {
1310 if (m_ndb)
1311 {
1312 Uint32 data[3];
1313 data[0]= NDB_LE_RestoreStarted;
1314 data[1]= backup_id;
1315 data[2]= node_id;
1316 Ndb_internal::send_event_report(false /* has lock */, m_ndb, data, 3);
1317 }
1318 return true;
1319 }
1320
1321 bool
report_meta_data(unsigned backup_id,unsigned node_id)1322 BackupRestore::report_meta_data(unsigned backup_id, unsigned node_id)
1323 {
1324 if (m_ndb)
1325 {
1326 Uint32 data[8];
1327 data[0]= NDB_LE_RestoreMetaData;
1328 data[1]= backup_id;
1329 data[2]= node_id;
1330 data[3]= m_n_tables;
1331 data[4]= m_n_tablespace;
1332 data[5]= m_n_logfilegroup;
1333 data[6]= m_n_datafile;
1334 data[7]= m_n_undofile;
1335 Ndb_internal::send_event_report(false /* has lock */, m_ndb, data, 8);
1336 }
1337 return true;
1338 }
1339 bool
report_data(unsigned backup_id,unsigned node_id)1340 BackupRestore::report_data(unsigned backup_id, unsigned node_id)
1341 {
1342 if (m_ndb)
1343 {
1344 Uint32 data[7];
1345 data[0]= NDB_LE_RestoreData;
1346 data[1]= backup_id;
1347 data[2]= node_id;
1348 data[3]= m_dataCount & 0xFFFFFFFF;
1349 data[4]= 0;
1350 data[5]= (Uint32)(m_dataBytes & 0xFFFFFFFF);
1351 data[6]= (Uint32)((m_dataBytes >> 32) & 0xFFFFFFFF);
1352 Ndb_internal::send_event_report(false /* has lock */, m_ndb, data, 7);
1353 }
1354 return true;
1355 }
1356
1357 bool
report_log(unsigned backup_id,unsigned node_id)1358 BackupRestore::report_log(unsigned backup_id, unsigned node_id)
1359 {
1360 if (m_ndb)
1361 {
1362 Uint32 data[7];
1363 data[0]= NDB_LE_RestoreLog;
1364 data[1]= backup_id;
1365 data[2]= node_id;
1366 data[3]= m_logCount & 0xFFFFFFFF;
1367 data[4]= 0;
1368 data[5]= (Uint32)(m_logBytes & 0xFFFFFFFF);
1369 data[6]= (Uint32)((m_logBytes >> 32) & 0xFFFFFFFF);
1370 Ndb_internal::send_event_report(false /* has lock */, m_ndb, data, 7);
1371 }
1372 return true;
1373 }
1374
1375 bool
report_completed(unsigned backup_id,unsigned node_id)1376 BackupRestore::report_completed(unsigned backup_id, unsigned node_id)
1377 {
1378 if (m_ndb)
1379 {
1380 Uint32 data[3];
1381 data[0]= NDB_LE_RestoreCompleted;
1382 data[1]= backup_id;
1383 data[2]= node_id;
1384 Ndb_internal::send_event_report(false /* has lock */, m_ndb, data, 3);
1385 }
1386 return true;
1387 }
1388
1389 bool
column_compatible_check(const char * tableName,const NDBCOL * backupCol,const NDBCOL * dbCol)1390 BackupRestore::column_compatible_check(const char* tableName,
1391 const NDBCOL* backupCol,
1392 const NDBCOL* dbCol)
1393 {
1394 if (backupCol->equal(*dbCol))
1395 return true;
1396
1397 /* Something is different between the columns, but some differences don't
1398 * matter.
1399 * Investigate which parts are different, and inform user
1400 */
1401 bool similarEnough = true;
1402
1403 /* We check similar things to NdbColumnImpl::equal() here */
1404 if (strcmp(backupCol->getName(), dbCol->getName()) != 0)
1405 {
1406 info << "Column " << tableName << "." << backupCol->getName()
1407 << " has different name in DB (" << dbCol->getName() << ")"
1408 << endl;
1409 similarEnough = false;
1410 }
1411
1412 if (backupCol->getType() != dbCol->getType())
1413 {
1414 info << "Column " << tableName << "." << backupCol->getName()
1415 << (" has different type in DB; promotion or lossy type conversion"
1416 " (demotion, signed/unsigned) may be required.") << endl;
1417 similarEnough = false;
1418 }
1419
1420 if (backupCol->getPrimaryKey() != dbCol->getPrimaryKey())
1421 {
1422 info << "Column " << tableName << "." << backupCol->getName()
1423 << (dbCol->getPrimaryKey()?" is":" is not")
1424 << " a primary key in the DB." << endl;
1425 similarEnough = false;
1426 }
1427 else
1428 {
1429 if (backupCol->getPrimaryKey())
1430 {
1431 if (backupCol->getDistributionKey() != dbCol->getDistributionKey())
1432 {
1433 info << "Column " << tableName << "." << backupCol->getName()
1434 << (dbCol->getDistributionKey()?" is":" is not")
1435 << " a distribution key in the DB." << endl;
1436 /* Not a problem for restore though */
1437 }
1438 }
1439 }
1440
1441 if (backupCol->getNullable() != dbCol->getNullable())
1442 {
1443 info << "Column " << tableName << "." << backupCol->getName()
1444 << (dbCol->getNullable()?" is":" is not")
1445 << " nullable in the DB." << endl;
1446 similarEnough = false;
1447 }
1448
1449 if (backupCol->getPrecision() != dbCol->getPrecision())
1450 {
1451 info << "Column " << tableName << "." << backupCol->getName()
1452 << " precision is different in the DB" << endl;
1453 similarEnough = false;
1454 }
1455
1456 if (backupCol->getScale() != dbCol->getScale())
1457 {
1458 info << "Column " << tableName << "." << backupCol->getName()
1459 << " scale is different in the DB" << endl;
1460 similarEnough = false;
1461 }
1462
1463 if (backupCol->getLength() != dbCol->getLength())
1464 {
1465 info << "Column " << tableName << "." << backupCol->getName()
1466 << " length is different in the DB" << endl;
1467 similarEnough = false;
1468 }
1469
1470 if (backupCol->getCharset() != dbCol->getCharset())
1471 {
1472 info << "Column " << tableName << "." << backupCol->getName()
1473 << " charset is different in the DB" << endl;
1474 similarEnough = false;
1475 }
1476
1477 if (backupCol->getAutoIncrement() != dbCol->getAutoIncrement())
1478 {
1479 info << "Column " << tableName << "." << backupCol->getName()
1480 << (dbCol->getAutoIncrement()?" is":" is not")
1481 << " AutoIncrementing in the DB" << endl;
1482 /* TODO : Can this be ignored? */
1483 similarEnough = false;
1484 }
1485
1486 {
1487 unsigned int backupDefaultLen, dbDefaultLen;
1488 const void *backupDefaultPtr, *dbDefaultPtr;
1489 backupDefaultPtr = backupCol->getDefaultValue(&backupDefaultLen);
1490 dbDefaultPtr = dbCol->getDefaultValue(&dbDefaultLen);
1491
1492 if ((backupDefaultLen != dbDefaultLen) ||
1493 (memcmp(backupDefaultPtr, dbDefaultPtr, backupDefaultLen) != 0))
1494 {
1495 info << "Column " << tableName << "." << backupCol->getName()
1496 << " Default value is different in the DB" << endl;
1497 /* This doesn't matter */
1498 }
1499 }
1500
1501 if (backupCol->getArrayType() != dbCol->getArrayType())
1502 {
1503 info << "Column " << tableName << "." << backupCol->getName()
1504 << " ArrayType is different in the DB" << endl;
1505 similarEnough = false;
1506 }
1507
1508 if (backupCol->getStorageType() != dbCol->getStorageType())
1509 {
1510 info << "Column " << tableName << "." << backupCol->getName()
1511 << " Storagetype is different in the DB" << endl;
1512 /* This doesn't matter */
1513 }
1514
1515 if (backupCol->getBlobVersion() != dbCol->getBlobVersion())
1516 {
1517 info << "Column " << tableName << "." << backupCol->getName()
1518 << " Blob version is different in the DB" << endl;
1519 similarEnough = false;
1520 }
1521
1522 if (backupCol->getDynamic() != dbCol->getDynamic())
1523 {
1524 info << "Column " << tableName << "." << backupCol->getName()
1525 << (dbCol->getDynamic()?" is":" is not")
1526 << " Dynamic in the DB" << endl;
1527 /* This doesn't matter */
1528 }
1529
1530 if (similarEnough)
1531 info << " Difference(s) will be ignored during restore." << endl;
1532 else
1533 info << " Difference(s) cannot be ignored. Cannot restore this column as is." << endl;
1534
1535 return similarEnough;
1536 }
1537
1538 bool
table_compatible_check(const TableS & tableS)1539 BackupRestore::table_compatible_check(const TableS & tableS)
1540 {
1541 if (!m_restore)
1542 return true;
1543
1544 const char *tablename = tableS.getTableName();
1545
1546 if(tableS.m_dictTable == NULL){
1547 ndbout<<"Table %s has no m_dictTable " << tablename << endl;
1548 return false;
1549 }
1550 /**
1551 * Ignore blob tables
1552 */
1553 if(match_blob(tablename) >= 0)
1554 return true;
1555
1556 const NdbTableImpl & tmptab = NdbTableImpl::getImpl(* tableS.m_dictTable);
1557 if ((int) tmptab.m_indexType != (int) NdbDictionary::Index::Undefined){
1558 return true;
1559 }
1560
1561 BaseString db_name, schema_name, table_name;
1562 if (!dissect_table_name(tablename, db_name, schema_name, table_name)) {
1563 return false;
1564 }
1565 check_rewrite_database(db_name);
1566
1567 m_ndb->setDatabaseName(db_name.c_str());
1568 m_ndb->setSchemaName(schema_name.c_str());
1569
1570 NdbDictionary::Dictionary* dict = m_ndb->getDictionary();
1571 const NdbDictionary::Table* tab = dict->getTable(table_name.c_str());
1572 if(tab == 0){
1573 err << "Unable to find table: " << table_name << endl;
1574 return false;
1575 }
1576
1577 /**
1578 * remap column(s) based on column-names
1579 */
1580 for (int i = 0; i<tableS.m_dictTable->getNoOfColumns(); i++)
1581 {
1582 AttributeDesc * attr_desc = tableS.getAttributeDesc(i);
1583 const NDBCOL * col_in_backup = tableS.m_dictTable->getColumn(i);
1584 const NDBCOL * col_in_kernel = tab->getColumn(col_in_backup->getName());
1585
1586 if (col_in_kernel == 0)
1587 {
1588 if ((m_tableChangesMask & TCM_EXCLUDE_MISSING_COLUMNS) == 0)
1589 {
1590 ndbout << "Missing column("
1591 << tableS.m_dictTable->getName() << "."
1592 << col_in_backup->getName()
1593 << ") in DB and exclude-missing-columns not specified" << endl;
1594 return false;
1595 }
1596
1597 info << "Column in backup ("
1598 << tableS.m_dictTable->getName() << "."
1599 << col_in_backup->getName()
1600 << ") missing in DB. Excluding column from restore." << endl;
1601
1602 attr_desc->m_exclude = true;
1603 }
1604 else
1605 {
1606 attr_desc->attrId = col_in_kernel->getColumnNo();
1607 }
1608 }
1609
1610 for (int i = 0; i<tab->getNoOfColumns(); i++)
1611 {
1612 const NDBCOL * col_in_kernel = tab->getColumn(i);
1613 const NDBCOL * col_in_backup =
1614 tableS.m_dictTable->getColumn(col_in_kernel->getName());
1615
1616 if (col_in_backup == 0)
1617 {
1618 if ((m_tableChangesMask & TCM_EXCLUDE_MISSING_COLUMNS) == 0)
1619 {
1620 ndbout << "Missing column("
1621 << tableS.m_dictTable->getName() << "."
1622 << col_in_kernel->getName()
1623 << ") in backup and exclude-missing-columns not specified"
1624 << endl;
1625 return false;
1626 }
1627
1628 /**
1629 * only nullable or defaulted non primary key columns can be missing from backup
1630 *
1631 */
1632 if (col_in_kernel->getPrimaryKey() ||
1633 ((col_in_kernel->getNullable() == false) &&
1634 (col_in_kernel->getDefaultValue() == NULL)))
1635 {
1636 ndbout << "Missing column("
1637 << tableS.m_dictTable->getName() << "."
1638 << col_in_kernel->getName()
1639 << ") in backup is primary key or not nullable or defaulted in DB"
1640 << endl;
1641 return false;
1642 }
1643
1644 info << "Column in DB ("
1645 << tableS.m_dictTable->getName() << "."
1646 << col_in_kernel->getName()
1647 << ") missing in Backup. Will be set to "
1648 << ((col_in_kernel->getDefaultValue() == NULL)?"Null":"Default value")
1649 << "." << endl;
1650 }
1651 }
1652
1653 AttrCheckCompatFunc attrCheckCompatFunc = NULL;
1654 for(int i = 0; i<tableS.m_dictTable->getNoOfColumns(); i++)
1655 {
1656 AttributeDesc * attr_desc = tableS.getAttributeDesc(i);
1657 if (attr_desc->m_exclude)
1658 continue;
1659
1660 const NDBCOL * col_in_kernel = tab->getColumn(attr_desc->attrId);
1661 const NDBCOL * col_in_backup = tableS.m_dictTable->getColumn(i);
1662
1663 if(column_compatible_check(tablename,
1664 col_in_backup,
1665 col_in_kernel))
1666 {
1667 continue;
1668 }
1669
1670 NDBCOL::Type type_in_backup = col_in_backup->getType();
1671 NDBCOL::Type type_in_kernel = col_in_kernel->getType();
1672 attrCheckCompatFunc = get_attr_check_compatability(type_in_backup,
1673 type_in_kernel);
1674 AttrConvType compat
1675 = (attrCheckCompatFunc == NULL ? ACT_UNSUPPORTED
1676 : attrCheckCompatFunc(*col_in_backup, *col_in_kernel));
1677 switch (compat) {
1678 case ACT_UNSUPPORTED:
1679 {
1680 err << "Table: "<< tablename
1681 << " column: " << col_in_backup->getName()
1682 << " incompatible with kernel's definition" << endl;
1683 return false;
1684 }
1685 case ACT_PRESERVING:
1686 if ((m_tableChangesMask & TCM_ATTRIBUTE_PROMOTION) == 0)
1687 {
1688 err << "Table: "<< tablename
1689 << " column: " << col_in_backup->getName()
1690 << " promotable to kernel's definition but option"
1691 << " promote-attributes not specified" << endl;
1692 return false;
1693 }
1694 break;
1695 case ACT_LOSSY:
1696 if ((m_tableChangesMask & TCM_ATTRIBUTE_DEMOTION) == 0)
1697 {
1698 err << "Table: "<< tablename
1699 << " column: " << col_in_backup->getName()
1700 << " convertable to kernel's definition but option"
1701 << " lossy-conversions not specified" << endl;
1702 return false;
1703 }
1704 break;
1705 default:
1706 err << "internal error: illegal value of compat = " << compat << endl;
1707 assert(false);
1708 return false;
1709 };
1710
1711 attr_desc->convertFunc = get_convert_func(type_in_backup,
1712 type_in_kernel);
1713 Uint32 m_attrSize = NdbColumnImpl::getImpl(*col_in_kernel).m_attrSize;
1714 Uint32 m_arraySize = NdbColumnImpl::getImpl(*col_in_kernel).m_arraySize;
1715
1716 // use a char_n_padding_struct to pass length information to convert()
1717 if (type_in_backup == NDBCOL::Char ||
1718 type_in_backup == NDBCOL::Binary ||
1719 type_in_backup == NDBCOL::Bit ||
1720 type_in_backup == NDBCOL::Varchar ||
1721 type_in_backup == NDBCOL::Longvarchar ||
1722 type_in_backup == NDBCOL::Varbinary ||
1723 type_in_backup == NDBCOL::Longvarbinary)
1724 {
1725 unsigned int size = sizeof(struct char_n_padding_struct) +
1726 m_attrSize * m_arraySize;
1727 struct char_n_padding_struct *s = (struct char_n_padding_struct *)
1728 malloc(size +2);
1729 if (!s)
1730 {
1731 err << "No more memory available!" << endl;
1732 exitHandler();
1733 }
1734 s->n_old = (attr_desc->size * attr_desc->arraySize) / 8;
1735 s->n_new = m_attrSize * m_arraySize;
1736 memset(s->new_row, 0 , m_attrSize * m_arraySize + 2);
1737 attr_desc->parameter = s;
1738 }
1739 else
1740 {
1741 unsigned int size = m_attrSize * m_arraySize;
1742 attr_desc->parameter = malloc(size + 2);
1743 if (!attr_desc->parameter)
1744 {
1745 err << "No more memory available!" << endl;
1746 exitHandler();
1747 }
1748 memset(attr_desc->parameter, 0, size + 2);
1749 }
1750
1751 info << "Data for column "
1752 << tablename << "."
1753 << col_in_backup->getName()
1754 << " will be converted from Backup type into DB type." << endl;
1755 }
1756
1757 return true;
1758 }
1759
1760 bool
createSystable(const TableS & tables)1761 BackupRestore::createSystable(const TableS & tables){
1762 if (!m_restore && !m_restore_meta && !m_restore_epoch)
1763 return true;
1764 const char *tablename = tables.getTableName();
1765
1766 if( strcmp(tablename, NDB_REP_DB "/def/" NDB_APPLY_TABLE) != 0 &&
1767 strcmp(tablename, NDB_REP_DB "/def/" NDB_SCHEMA_TABLE) != 0 )
1768 {
1769 return true;
1770 }
1771
1772 BaseString db_name, schema_name, table_name;
1773 if (!dissect_table_name(tablename, db_name, schema_name, table_name)) {
1774 return false;
1775 }
1776 // do not rewrite database for system tables:
1777 // check_rewrite_database(db_name);
1778
1779 m_ndb->setDatabaseName(db_name.c_str());
1780 m_ndb->setSchemaName(schema_name.c_str());
1781
1782 NdbDictionary::Dictionary* dict = m_ndb->getDictionary();
1783 if( dict->getTable(table_name.c_str()) != NULL ){
1784 return true;
1785 }
1786 return table(tables);
1787 }
1788
1789 bool
table(const TableS & table)1790 BackupRestore::table(const TableS & table){
1791 if (!m_restore && !m_restore_meta && !m_rebuild_indexes && !m_disable_indexes)
1792 return true;
1793
1794 const char * name = table.getTableName();
1795
1796 /**
1797 * Ignore blob tables
1798 */
1799 if(match_blob(name) >= 0)
1800 return true;
1801
1802 const NdbTableImpl & tmptab = NdbTableImpl::getImpl(* table.m_dictTable);
1803 if ((int) tmptab.m_indexType != (int) NdbDictionary::Index::Undefined){
1804 m_indexes.push_back(table.m_dictTable);
1805 return true;
1806 }
1807
1808 BaseString db_name, schema_name, table_name;
1809 if (!dissect_table_name(name, db_name, schema_name, table_name)) {
1810 return false;
1811 }
1812 check_rewrite_database(db_name);
1813
1814 m_ndb->setDatabaseName(db_name.c_str());
1815 m_ndb->setSchemaName(schema_name.c_str());
1816
1817 NdbDictionary::Dictionary* dict = m_ndb->getDictionary();
1818 if(m_restore_meta)
1819 {
1820 NdbDictionary::Table copy(*table.m_dictTable);
1821
1822 copy.setName(table_name.c_str());
1823 Uint32 id;
1824 if (copy.getTablespace(&id))
1825 {
1826 debug << "Connecting " << name << " to tablespace oldid: " << id << flush;
1827 NdbDictionary::Tablespace* ts = m_tablespaces[id];
1828 debug << " newid: " << ts->getObjectId() << endl;
1829 copy.setTablespace(* ts);
1830 }
1831
1832 if (copy.getFragmentType() == NdbDictionary::Object::HashMapPartition)
1833 {
1834 Uint32 id;
1835 if (copy.getHashMap(&id))
1836 {
1837 NdbDictionary::HashMap * hm = m_hashmaps[id];
1838 copy.setHashMap(* hm);
1839 }
1840 }
1841 else if (copy.getDefaultNoPartitionsFlag())
1842 {
1843 /*
1844 Table was defined with default number of partitions. We can restore
1845 it with whatever is the default in this cluster.
1846 We use the max_rows parameter in calculating the default number.
1847 */
1848 Uint32 no_nodes = m_cluster_connection->no_db_nodes();
1849 copy.setFragmentCount(get_no_fragments(copy.getMaxRows(),
1850 no_nodes));
1851 set_default_nodegroups(©);
1852 }
1853 else
1854 {
1855 /*
1856 Table was defined with specific number of partitions. It should be
1857 restored with the same number of partitions. It will either be
1858 restored in the same node groups as when backup was taken or by
1859 using a node group map supplied to the ndb_restore program.
1860 */
1861 Vector<Uint32> new_array;
1862 Uint16 no_parts = copy.getFragmentCount();
1863 new_array.assign(copy.getFragmentData(), no_parts);
1864 if (map_nodegroups(new_array.getBase(), no_parts))
1865 {
1866 if (translate_frm(©))
1867 {
1868 err << "Create table " << table.getTableName() << " failed: ";
1869 err << "Translate frm error" << endl;
1870 return false;
1871 }
1872 }
1873 copy.setFragmentData(new_array.getBase(), no_parts);
1874 }
1875
1876 /**
1877 * Force of varpart was introduced in 5.1.18, telco 6.1.7 and 6.2.1
1878 * Since default from mysqld is to add force of varpart (disable with
1879 * ROW_FORMAT=FIXED) we force varpart onto tables when they are restored
1880 * from backups taken with older versions. This will be wrong if
1881 * ROW_FORMAT=FIXED was used on original table, however the likelyhood of
1882 * this is low, since ROW_FORMAT= was a NOOP in older versions.
1883 */
1884
1885 if (table.getBackupVersion() < MAKE_VERSION(5,1,18))
1886 copy.setForceVarPart(true);
1887 else if (getMajor(table.getBackupVersion()) == 6 &&
1888 (table.getBackupVersion() < MAKE_VERSION(6,1,7) ||
1889 table.getBackupVersion() == MAKE_VERSION(6,2,0)))
1890 copy.setForceVarPart(true);
1891
1892 /*
1893 update min and max rows to reflect the table, this to
1894 ensure that memory is allocated properly in the ndb kernel
1895 */
1896 copy.setMinRows(table.getNoOfRecords());
1897 if (table.getNoOfRecords() > copy.getMaxRows())
1898 {
1899 copy.setMaxRows(table.getNoOfRecords());
1900 }
1901
1902 NdbTableImpl &tableImpl = NdbTableImpl::getImpl(copy);
1903 if (table.getBackupVersion() < MAKE_VERSION(5,1,0) && !m_no_upgrade){
1904 for(int i= 0; i < copy.getNoOfColumns(); i++)
1905 {
1906 NdbDictionary::Column::Type t = copy.getColumn(i)->getType();
1907
1908 if (t == NdbDictionary::Column::Varchar ||
1909 t == NdbDictionary::Column::Varbinary)
1910 tableImpl.getColumn(i)->setArrayType(NdbDictionary::Column::ArrayTypeShortVar);
1911 if (t == NdbDictionary::Column::Longvarchar ||
1912 t == NdbDictionary::Column::Longvarbinary)
1913 tableImpl.getColumn(i)->setArrayType(NdbDictionary::Column::ArrayTypeMediumVar);
1914 }
1915 }
1916
1917 if (dict->createTable(copy) == -1)
1918 {
1919 err << "Create table `" << table.getTableName() << "` failed: "
1920 << dict->getNdbError() << endl;
1921 if (dict->getNdbError().code == 771)
1922 {
1923 /*
1924 The user on the cluster where the backup was created had specified
1925 specific node groups for partitions. Some of these node groups
1926 didn't exist on this cluster. We will warn the user of this and
1927 inform him of his option.
1928 */
1929 err << "The node groups defined in the table didn't exist in this";
1930 err << " cluster." << endl << "There is an option to use the";
1931 err << " the parameter ndb-nodegroup-map to define a mapping from";
1932 err << endl << "the old nodegroups to new nodegroups" << endl;
1933 }
1934 return false;
1935 }
1936 info.setLevel(254);
1937 info << "Successfully restored table `"
1938 << table.getTableName() << "`" << endl;
1939 }
1940
1941 const NdbDictionary::Table* tab = dict->getTable(table_name.c_str());
1942 if(tab == 0){
1943 err << "Unable to find table: `" << table_name << "`" << endl;
1944 return false;
1945 }
1946 if(m_restore_meta)
1947 {
1948 if (tab->getFrmData())
1949 {
1950 // a MySQL Server table is restored, thus an event should be created
1951 BaseString event_name("REPL$");
1952 event_name.append(db_name.c_str());
1953 event_name.append("/");
1954 event_name.append(table_name.c_str());
1955
1956 NdbDictionary::Event my_event(event_name.c_str());
1957 my_event.setTable(*tab);
1958 my_event.addTableEvent(NdbDictionary::Event::TE_ALL);
1959 my_event.setReport(NdbDictionary::Event::ER_DDL);
1960
1961 // add all columns to the event
1962 bool has_blobs = false;
1963 for(int a= 0; a < tab->getNoOfColumns(); a++)
1964 {
1965 my_event.addEventColumn(a);
1966 NdbDictionary::Column::Type t = tab->getColumn(a)->getType();
1967 if (t == NdbDictionary::Column::Blob ||
1968 t == NdbDictionary::Column::Text)
1969 has_blobs = true;
1970 }
1971 if (has_blobs)
1972 my_event.mergeEvents(true);
1973
1974 while ( dict->createEvent(my_event) ) // Add event to database
1975 {
1976 if (dict->getNdbError().classification == NdbError::SchemaObjectExists)
1977 {
1978 info << "Event for table " << table.getTableName()
1979 << " already exists, removing.\n";
1980 if (!dict->dropEvent(my_event.getName(), 1))
1981 continue;
1982 }
1983 err << "Create table event for " << table.getTableName() << " failed: "
1984 << dict->getNdbError() << endl;
1985 dict->dropTable(table_name.c_str());
1986 return false;
1987 }
1988 info.setLevel(254);
1989 info << "Successfully restored table event " << event_name << endl ;
1990 }
1991 }
1992 const NdbDictionary::Table* null = 0;
1993 m_new_tables.fill(table.m_dictTable->getTableId(), null);
1994 m_new_tables[table.m_dictTable->getTableId()] = tab;
1995
1996 m_n_tables++;
1997
1998 return true;
1999 }
2000
2001 bool
endOfTables()2002 BackupRestore::endOfTables(){
2003 if(!m_restore_meta && !m_rebuild_indexes && !m_disable_indexes)
2004 return true;
2005
2006 NdbDictionary::Dictionary* dict = m_ndb->getDictionary();
2007 for(size_t i = 0; i<m_indexes.size(); i++){
2008 NdbTableImpl & indtab = NdbTableImpl::getImpl(* m_indexes[i]);
2009
2010 BaseString db_name, schema_name, table_name;
2011 if (!dissect_table_name(indtab.m_primaryTable.c_str(),
2012 db_name, schema_name, table_name)) {
2013 return false;
2014 }
2015 check_rewrite_database(db_name);
2016
2017 m_ndb->setDatabaseName(db_name.c_str());
2018 m_ndb->setSchemaName(schema_name.c_str());
2019
2020 const NdbDictionary::Table * prim = dict->getTable(table_name.c_str());
2021 if(prim == 0){
2022 err << "Unable to find base table `" << table_name
2023 << "` for index `"
2024 << indtab.getName() << "`" << endl;
2025 if (ga_skip_broken_objects)
2026 {
2027 continue;
2028 }
2029 return false;
2030 }
2031 NdbTableImpl& base = NdbTableImpl::getImpl(*prim);
2032 NdbIndexImpl* idx;
2033 Vector<BaseString> split_idx;
2034 {
2035 BaseString tmp(indtab.getName());
2036 if (tmp.split(split_idx, "/") != 4)
2037 {
2038 err << "Invalid index name format `" << indtab.getName() << "`" << endl;
2039 return false;
2040 }
2041 }
2042 if(NdbDictInterface::create_index_obj_from_table(&idx, &indtab, &base))
2043 {
2044 err << "Failed to create index `" << split_idx[3]
2045 << "` on " << table_name << endl;
2046 return false;
2047 }
2048 idx->setName(split_idx[3].c_str());
2049 if (m_restore_meta && !m_disable_indexes && !m_rebuild_indexes)
2050 {
2051 if (dict->createIndex(* idx) != 0)
2052 {
2053 delete idx;
2054 err << "Failed to create index `" << split_idx[3].c_str()
2055 << "` on `" << table_name << "`" << endl
2056 << dict->getNdbError() << endl;
2057
2058 return false;
2059 }
2060 info << "Successfully created index `" << split_idx[3].c_str()
2061 << "` on `" << table_name << "`" << endl;
2062 }
2063 else if (m_disable_indexes)
2064 {
2065 int res = dict->dropIndex(idx->getName(), prim->getName());
2066 if (res == 0)
2067 {
2068 info << "Dropped index `" << split_idx[3].c_str()
2069 << "` on `" << table_name << "`" << endl;
2070 }
2071 }
2072 Uint32 id = prim->getObjectId();
2073 if (m_index_per_table.size() <= id)
2074 {
2075 Vector<NdbDictionary::Index*> tmp;
2076 m_index_per_table.fill(id + 1, tmp);
2077 }
2078 Vector<NdbDictionary::Index*> & list = m_index_per_table[id];
2079 list.push_back(idx);
2080 }
2081 return true;
2082 }
2083
tuple(const TupleS & tup,Uint32 fragmentId)2084 void BackupRestore::tuple(const TupleS & tup, Uint32 fragmentId)
2085 {
2086 const TableS * tab = tup.getTable();
2087
2088 if (!m_restore)
2089 return;
2090
2091 while (m_free_callback == 0)
2092 {
2093 assert(m_transactions == m_parallelism);
2094 // send-poll all transactions
2095 // close transaction is done in callback
2096 m_ndb->sendPollNdb(3000, 1);
2097 }
2098
2099 restore_callback_t * cb = m_free_callback;
2100
2101 if (cb == 0)
2102 assert(false);
2103
2104 cb->retries = 0;
2105 cb->fragId = fragmentId;
2106 cb->tup = tup; // must do copy!
2107
2108 if (tab->isSYSTAB_0())
2109 {
2110 tuple_SYSTAB_0(cb, *tab);
2111 return;
2112 }
2113
2114 m_free_callback = cb->next;
2115
2116 tuple_a(cb);
2117 }
2118
tuple_a(restore_callback_t * cb)2119 void BackupRestore::tuple_a(restore_callback_t *cb)
2120 {
2121 Uint32 partition_id = cb->fragId;
2122 Uint32 n_bytes;
2123 while (cb->retries < 10)
2124 {
2125 /**
2126 * start transactions
2127 */
2128 cb->connection = m_ndb->startTransaction();
2129 if (cb->connection == NULL)
2130 {
2131 if (errorHandler(cb))
2132 {
2133 m_ndb->sendPollNdb(3000, 1);
2134 continue;
2135 }
2136 err << "Cannot start transaction" << endl;
2137 exitHandler();
2138 } // if
2139
2140 const TupleS &tup = cb->tup;
2141 const NdbDictionary::Table * table = get_table(tup.getTable()->m_dictTable);
2142
2143 NdbOperation * op = cb->connection->getNdbOperation(table);
2144
2145 if (op == NULL)
2146 {
2147 if (errorHandler(cb))
2148 continue;
2149 err << "Cannot get operation: " << cb->connection->getNdbError() << endl;
2150 exitHandler();
2151 } // if
2152
2153 if (op->writeTuple() == -1)
2154 {
2155 if (errorHandler(cb))
2156 continue;
2157 err << "Error defining op: " << cb->connection->getNdbError() << endl;
2158 exitHandler();
2159 } // if
2160
2161 n_bytes= 0;
2162
2163 if (table->getFragmentType() == NdbDictionary::Object::UserDefined)
2164 {
2165 if (table->getDefaultNoPartitionsFlag())
2166 {
2167 /*
2168 This can only happen for HASH partitioning with
2169 user defined hash function where user hasn't
2170 specified the number of partitions and we
2171 have to calculate it. We use the hash value
2172 stored in the record to calculate the partition
2173 to use.
2174 */
2175 int i = tup.getNoOfAttributes() - 1;
2176 const AttributeData *attr_data = tup.getData(i);
2177 Uint32 hash_value = *attr_data->u_int32_value;
2178 op->setPartitionId(get_part_id(table, hash_value));
2179 }
2180 else
2181 {
2182 /*
2183 Either RANGE or LIST (with or without subparts)
2184 OR HASH partitioning with user defined hash
2185 function but with fixed set of partitions.
2186 */
2187 op->setPartitionId(partition_id);
2188 }
2189 }
2190 int ret = 0;
2191 for (int j = 0; j < 2; j++)
2192 {
2193 for (int i = 0; i < tup.getNoOfAttributes(); i++)
2194 {
2195 AttributeDesc * attr_desc = tup.getDesc(i);
2196 const AttributeData * attr_data = tup.getData(i);
2197 int size = attr_desc->size;
2198 int arraySize = attr_desc->arraySize;
2199 char * dataPtr = attr_data->string_value;
2200 Uint32 length = 0;
2201
2202 if (attr_desc->m_exclude)
2203 continue;
2204
2205 if (!attr_data->null)
2206 {
2207 const unsigned char * src = (const unsigned char *)dataPtr;
2208 switch(attr_desc->m_column->getType()){
2209 case NdbDictionary::Column::Varchar:
2210 case NdbDictionary::Column::Varbinary:
2211 length = src[0] + 1;
2212 break;
2213 case NdbDictionary::Column::Longvarchar:
2214 case NdbDictionary::Column::Longvarbinary:
2215 length = src[0] + (src[1] << 8) + 2;
2216 break;
2217 default:
2218 length = attr_data->size;
2219 break;
2220 }
2221 }
2222 if (j == 0 && tup.getTable()->have_auto_inc(i))
2223 tup.getTable()->update_max_auto_val(dataPtr,size*arraySize);
2224
2225 if (attr_desc->convertFunc)
2226 {
2227 if ((attr_desc->m_column->getPrimaryKey() && j == 0) ||
2228 (j == 1 && !attr_data->null))
2229 {
2230 bool truncated = true; // assume data truncation until overridden
2231 dataPtr = (char*)attr_desc->convertFunc(dataPtr,
2232 attr_desc->parameter,
2233 truncated);
2234 if (!dataPtr)
2235 {
2236 err << "Error: Convert data failed when restoring tuples!" << endl;
2237 exitHandler();
2238 }
2239 if (truncated)
2240 {
2241 // wl5421: option to report data truncation on tuple of desired
2242 //err << "====== data truncation detected for column: "
2243 // << attr_desc->m_column->getName() << endl;
2244 attr_desc->truncation_detected = true;
2245 }
2246 }
2247 }
2248
2249 if (attr_desc->m_column->getPrimaryKey())
2250 {
2251 if (j == 1) continue;
2252 ret = op->equal(attr_desc->attrId, dataPtr, length);
2253 }
2254 else
2255 {
2256 if (j == 0) continue;
2257 if (attr_data->null)
2258 ret = op->setValue(attr_desc->attrId, NULL, 0);
2259 else
2260 ret = op->setValue(attr_desc->attrId, dataPtr, length);
2261 }
2262 if (ret < 0) {
2263 ndbout_c("Column: %d type %d %d %d %d",i,
2264 attr_desc->m_column->getType(),
2265 size, arraySize, length);
2266 break;
2267 }
2268 n_bytes+= length;
2269 }
2270 if (ret < 0)
2271 break;
2272 }
2273 if (ret < 0)
2274 {
2275 if (errorHandler(cb))
2276 continue;
2277 err << "Error defining op: " << cb->connection->getNdbError() << endl;
2278 exitHandler();
2279 }
2280
2281 if (opt_no_binlog)
2282 {
2283 op->setAnyValue(NDB_ANYVALUE_FOR_NOLOGGING);
2284 }
2285
2286 // Prepare transaction (the transaction is NOT yet sent to NDB)
2287 cb->n_bytes= n_bytes;
2288 cb->connection->executeAsynchPrepare(NdbTransaction::Commit,
2289 &callback, cb);
2290 m_transactions++;
2291 return;
2292 }
2293 err << "Retried transaction " << cb->retries << " times.\nLast error"
2294 << m_ndb->getNdbError(cb->error_code) << endl
2295 << "...Unable to recover from errors. Exiting..." << endl;
2296 exitHandler();
2297 }
2298
tuple_SYSTAB_0(restore_callback_t * cb,const TableS & tab)2299 void BackupRestore::tuple_SYSTAB_0(restore_callback_t *cb,
2300 const TableS & tab)
2301 {
2302 const TupleS & tup = cb->tup;
2303 Uint32 syskey;
2304 Uint64 nextid;
2305
2306 if (tab.get_auto_data(tup, &syskey, &nextid))
2307 {
2308 /*
2309 We found a valid auto_increment value in SYSTAB_0
2310 where syskey is a table_id and nextid is next auto_increment
2311 value.
2312 */
2313 if (restoreAutoIncrement(cb, syskey, nextid) == -1)
2314 exitHandler();
2315 }
2316 }
2317
restoreAutoIncrement(restore_callback_t * cb,Uint32 tableId,Uint64 value)2318 int BackupRestore::restoreAutoIncrement(restore_callback_t *cb,
2319 Uint32 tableId, Uint64 value)
2320 {
2321 /*
2322 Restore the auto_increment value found in SYSTAB_0 from
2323 backup. First map the old table id to the new table while
2324 also checking that it is an actual table will some auto_increment
2325 column. Note that the SYSTAB_0 table in the backup can contain
2326 stale information from dropped tables.
2327 */
2328 int result = 0;
2329 const NdbDictionary::Table* tab = (tableId < m_new_tables.size())? m_new_tables[tableId] : NULL;
2330 if (tab && tab->getNoOfAutoIncrementColumns() > 0)
2331 {
2332 /*
2333 Write the auto_increment value back into SYSTAB_0.
2334 This is done in a separate transaction and could possibly
2335 fail, so we retry if a temporary error is received.
2336 */
2337 while (cb->retries < 10)
2338 {
2339 if ((result = m_ndb->setAutoIncrementValue(tab, value, false) == -1))
2340 {
2341 if (errorHandler(cb))
2342 {
2343 continue;
2344 }
2345 }
2346 break;
2347 }
2348 }
2349 return result;
2350 }
2351
cback(int result,restore_callback_t * cb)2352 void BackupRestore::cback(int result, restore_callback_t *cb)
2353 {
2354 m_transactions--;
2355
2356 if (result < 0)
2357 {
2358 /**
2359 * Error. temporary or permanent?
2360 */
2361 if (errorHandler(cb))
2362 tuple_a(cb); // retry
2363 else
2364 {
2365 err << "Restore: Failed to restore data due to a unrecoverable error. Exiting..." << endl;
2366 exitHandler();
2367 }
2368 }
2369 else
2370 {
2371 /**
2372 * OK! close transaction
2373 */
2374 m_ndb->closeTransaction(cb->connection);
2375 cb->connection= 0;
2376 cb->next= m_free_callback;
2377 m_free_callback= cb;
2378 m_dataBytes+= cb->n_bytes;
2379 m_dataCount++;
2380 }
2381 }
2382
2383 /**
2384 * returns true if is recoverable,
2385 * Error handling based on hugo
2386 * false if it is an error that generates an abort.
2387 */
errorHandler(restore_callback_t * cb)2388 bool BackupRestore::errorHandler(restore_callback_t *cb)
2389 {
2390 NdbError error;
2391 if(cb->connection)
2392 {
2393 error= cb->connection->getNdbError();
2394 m_ndb->closeTransaction(cb->connection);
2395 cb->connection= 0;
2396 }
2397 else
2398 {
2399 error= m_ndb->getNdbError();
2400 }
2401
2402 Uint32 sleepTime = 100 + cb->retries * 300;
2403
2404 cb->retries++;
2405 cb->error_code = error.code;
2406
2407 switch(error.status)
2408 {
2409 case NdbError::Success:
2410 err << "Success error: " << error << endl;
2411 return false;
2412 // ERROR!
2413
2414 case NdbError::TemporaryError:
2415 err << "Temporary error: " << error << endl;
2416 m_temp_error = true;
2417 NdbSleep_MilliSleep(sleepTime);
2418 return true;
2419 // RETRY
2420
2421 case NdbError::UnknownResult:
2422 err << "Unknown: " << error << endl;
2423 return false;
2424 // ERROR!
2425
2426 default:
2427 case NdbError::PermanentError:
2428 //ERROR
2429 err << "Permanent: " << error << endl;
2430 return false;
2431 }
2432 err << "No error status" << endl;
2433 return false;
2434 }
2435
exitHandler()2436 void BackupRestore::exitHandler()
2437 {
2438 release();
2439 NDBT_ProgramExit(NDBT_FAILED);
2440 exit(NDBT_FAILED);
2441 }
2442
2443
2444 void
tuple_free()2445 BackupRestore::tuple_free()
2446 {
2447 if (!m_restore)
2448 return;
2449
2450 // Poll all transactions
2451 while (m_transactions)
2452 {
2453 m_ndb->sendPollNdb(3000);
2454 }
2455 }
2456
2457 void
endOfTuples()2458 BackupRestore::endOfTuples()
2459 {
2460 tuple_free();
2461 }
2462
2463 #ifdef NOT_USED
use_part_id(const NdbDictionary::Table * table)2464 static bool use_part_id(const NdbDictionary::Table *table)
2465 {
2466 if (table->getDefaultNoPartitionsFlag() &&
2467 (table->getFragmentType() == NdbDictionary::Object::UserDefined))
2468 return false;
2469 else
2470 return true;
2471 }
2472 #endif
2473
get_part_id(const NdbDictionary::Table * table,Uint32 hash_value)2474 static Uint32 get_part_id(const NdbDictionary::Table *table,
2475 Uint32 hash_value)
2476 {
2477 Uint32 no_frags = table->getFragmentCount();
2478
2479 if (table->getLinearFlag())
2480 {
2481 Uint32 part_id;
2482 Uint32 mask = 1;
2483 while (no_frags > mask) mask <<= 1;
2484 mask--;
2485 part_id = hash_value & mask;
2486 if (part_id >= no_frags)
2487 part_id = hash_value & (mask >> 1);
2488 return part_id;
2489 }
2490 else
2491 return (hash_value % no_frags);
2492 }
2493
2494 struct TransGuard
2495 {
2496 NdbTransaction* pTrans;
TransGuardTransGuard2497 TransGuard(NdbTransaction* p) : pTrans(p) {}
~TransGuardTransGuard2498 ~TransGuard() { if (pTrans) pTrans->close();}
2499 };
2500
2501 void
logEntry(const LogEntry & tup)2502 BackupRestore::logEntry(const LogEntry & tup)
2503 {
2504 if (!m_restore)
2505 return;
2506
2507
2508 Uint32 retries = 0;
2509 NdbError errobj;
2510 retry:
2511 if (retries == 11)
2512 {
2513 err << "execute failed: " << errobj << endl;
2514 exitHandler();
2515 }
2516 else if (retries > 0)
2517 {
2518 NdbSleep_MilliSleep(100 + (retries - 1) * 100);
2519 }
2520
2521 retries++;
2522
2523 NdbTransaction * trans = m_ndb->startTransaction();
2524 if (trans == NULL)
2525 {
2526 errobj = m_ndb->getNdbError();
2527 if (errobj.status == NdbError::TemporaryError)
2528 {
2529 goto retry;
2530 }
2531 err << "Cannot start transaction: " << errobj << endl;
2532 exitHandler();
2533 } // if
2534
2535 TransGuard g(trans);
2536 const NdbDictionary::Table * table = get_table(tup.m_table->m_dictTable);
2537 NdbOperation * op = trans->getNdbOperation(table);
2538 if (op == NULL)
2539 {
2540 err << "Cannot get operation: " << trans->getNdbError() << endl;
2541 exitHandler();
2542 } // if
2543
2544 int check = 0;
2545 switch(tup.m_type)
2546 {
2547 case LogEntry::LE_INSERT:
2548 check = op->insertTuple();
2549 break;
2550 case LogEntry::LE_UPDATE:
2551 check = op->updateTuple();
2552 break;
2553 case LogEntry::LE_DELETE:
2554 check = op->deleteTuple();
2555 break;
2556 default:
2557 err << "Log entry has wrong operation type."
2558 << " Exiting...";
2559 exitHandler();
2560 }
2561
2562 if (check != 0)
2563 {
2564 err << "Error defining op: " << trans->getNdbError() << endl;
2565 exitHandler();
2566 } // if
2567
2568 if (table->getFragmentType() == NdbDictionary::Object::UserDefined)
2569 {
2570 if (table->getDefaultNoPartitionsFlag())
2571 {
2572 const AttributeS * attr = tup[tup.size()-1];
2573 Uint32 hash_value = *(Uint32*)attr->Data.string_value;
2574 op->setPartitionId(get_part_id(table, hash_value));
2575 }
2576 else
2577 op->setPartitionId(tup.m_frag_id);
2578 }
2579
2580 Bitmask<4096> keys;
2581 Uint32 n_bytes= 0;
2582 for (Uint32 i= 0; i < tup.size(); i++)
2583 {
2584 const AttributeS * attr = tup[i];
2585 int size = attr->Desc->size;
2586 int arraySize = attr->Desc->arraySize;
2587 const char * dataPtr = attr->Data.string_value;
2588
2589 if (attr->Desc->m_exclude)
2590 continue;
2591
2592 if (tup.m_table->have_auto_inc(attr->Desc->attrId))
2593 tup.m_table->update_max_auto_val(dataPtr,size*arraySize);
2594
2595 const Uint32 length = (size / 8) * arraySize;
2596 n_bytes+= length;
2597
2598 if (attr->Desc->convertFunc)
2599 {
2600 bool truncated = true; // assume data truncation until overridden
2601 dataPtr = (char*)attr->Desc->convertFunc(dataPtr,
2602 attr->Desc->parameter,
2603 truncated);
2604 if (!dataPtr)
2605 {
2606 err << "Error: Convert data failed when restoring tuples!" << endl;
2607 exitHandler();
2608 }
2609 if (truncated)
2610 {
2611 // wl5421: option to report data truncation on tuple of desired
2612 //err << "****** data truncation detected for column: "
2613 // << attr->Desc->m_column->getName() << endl;
2614 attr->Desc->truncation_detected = true;
2615 }
2616 }
2617
2618 if (attr->Desc->m_column->getPrimaryKey())
2619 {
2620 if(!keys.get(attr->Desc->attrId))
2621 {
2622 keys.set(attr->Desc->attrId);
2623 check= op->equal(attr->Desc->attrId, dataPtr, length);
2624 }
2625 }
2626 else
2627 check= op->setValue(attr->Desc->attrId, dataPtr, length);
2628
2629 if (check != 0)
2630 {
2631 err << "Error defining op: " << trans->getNdbError() << endl;
2632 exitHandler();
2633 } // if
2634 }
2635
2636 if (opt_no_binlog)
2637 {
2638 op->setAnyValue(NDB_ANYVALUE_FOR_NOLOGGING);
2639 }
2640 const int ret = trans->execute(NdbTransaction::Commit);
2641 if (ret != 0)
2642 {
2643 // Both insert update and delete can fail during log running
2644 // and it's ok
2645 bool ok= false;
2646 errobj= trans->getNdbError();
2647 if (errobj.status == NdbError::TemporaryError)
2648 goto retry;
2649
2650 switch(tup.m_type)
2651 {
2652 case LogEntry::LE_INSERT:
2653 if(errobj.status == NdbError::PermanentError &&
2654 errobj.classification == NdbError::ConstraintViolation)
2655 ok= true;
2656 break;
2657 case LogEntry::LE_UPDATE:
2658 case LogEntry::LE_DELETE:
2659 if(errobj.status == NdbError::PermanentError &&
2660 errobj.classification == NdbError::NoDataFound)
2661 ok= true;
2662 break;
2663 }
2664 if (!ok)
2665 {
2666 err << "execute failed: " << errobj << endl;
2667 exitHandler();
2668 }
2669 }
2670
2671 m_logBytes+= n_bytes;
2672 m_logCount++;
2673 }
2674
2675 void
endOfLogEntrys()2676 BackupRestore::endOfLogEntrys()
2677 {
2678 if (!m_restore)
2679 return;
2680
2681 info.setLevel(254);
2682 info << "Restored " << m_dataCount << " tuples and "
2683 << m_logCount << " log entries" << endl;
2684 }
2685
2686 /*
2687 * callback : This is called when the transaction is polled
2688 *
2689 * (This function must have three arguments:
2690 * - The result of the transaction,
2691 * - The NdbTransaction object, and
2692 * - A pointer to an arbitrary object.)
2693 */
2694
2695 static void
callback(int result,NdbTransaction * trans,void * aObject)2696 callback(int result, NdbTransaction* trans, void* aObject)
2697 {
2698 restore_callback_t *cb = (restore_callback_t *)aObject;
2699 (cb->restore)->cback(result, cb);
2700 }
2701
2702
2703 AttrCheckCompatFunc
get_attr_check_compatability(const NDBCOL::Type & old_type,const NDBCOL::Type & new_type)2704 BackupRestore::get_attr_check_compatability(const NDBCOL::Type &old_type,
2705 const NDBCOL::Type &new_type)
2706 {
2707 int i = 0;
2708 NDBCOL::Type first_item = m_allowed_promotion_attrs[0].old_type;
2709 NDBCOL::Type second_item = m_allowed_promotion_attrs[0].new_type;
2710
2711 while (first_item != old_type || second_item != new_type)
2712 {
2713 if (first_item == NDBCOL::Undefined)
2714 break;
2715
2716 i++;
2717 first_item = m_allowed_promotion_attrs[i].old_type;
2718 second_item = m_allowed_promotion_attrs[i].new_type;
2719 }
2720 if (first_item == old_type && second_item == new_type)
2721 return m_allowed_promotion_attrs[i].attr_check_compatability;
2722 return NULL;
2723 }
2724
2725 AttrConvertFunc
get_convert_func(const NDBCOL::Type & old_type,const NDBCOL::Type & new_type)2726 BackupRestore::get_convert_func(const NDBCOL::Type &old_type,
2727 const NDBCOL::Type &new_type)
2728 {
2729 int i = 0;
2730 NDBCOL::Type first_item = m_allowed_promotion_attrs[0].old_type;
2731 NDBCOL::Type second_item = m_allowed_promotion_attrs[0].new_type;
2732
2733 while (first_item != old_type || second_item != new_type)
2734 {
2735 if (first_item == NDBCOL::Undefined)
2736 break;
2737 i++;
2738 first_item = m_allowed_promotion_attrs[i].old_type;
2739 second_item = m_allowed_promotion_attrs[i].new_type;
2740 }
2741 if (first_item == old_type && second_item == new_type)
2742 return m_allowed_promotion_attrs[i].attr_convert;
2743
2744 return NULL;
2745
2746 }
2747
2748 AttrConvType
check_compat_promotion(const NDBCOL & old_col,const NDBCOL & new_col)2749 BackupRestore::check_compat_promotion(const NDBCOL &old_col,
2750 const NDBCOL &new_col)
2751 {
2752 return ACT_PRESERVING;
2753 }
2754
2755 AttrConvType
check_compat_lossy(const NDBCOL & old_col,const NDBCOL & new_col)2756 BackupRestore::check_compat_lossy(const NDBCOL &old_col,
2757 const NDBCOL &new_col)
2758 {
2759 return ACT_LOSSY;
2760 }
2761
2762 AttrConvType
check_compat_sizes(const NDBCOL & old_col,const NDBCOL & new_col)2763 BackupRestore::check_compat_sizes(const NDBCOL &old_col,
2764 const NDBCOL &new_col)
2765 {
2766 // the size (width) of the element type
2767 Uint32 new_size = new_col.getSize();
2768 Uint32 old_size = old_col.getSize();
2769 // the fixed/max array length (1 for scalars)
2770 Uint32 new_length = new_col.getLength();
2771 Uint32 old_length = old_col.getLength();
2772
2773 // identity conversions have been handled by column_compatible_check()
2774 assert(new_size != old_size
2775 || new_length != old_length
2776 || new_col.getArrayType() != old_col.getArrayType());
2777
2778 // test for loss of element width or array length
2779 if (new_size < old_size || new_length < old_length) {
2780 return ACT_LOSSY;
2781 }
2782
2783 // not tested: conversions varying in both, array length and element width
2784 if (new_size != old_size && new_length != old_length) {
2785 return ACT_UNSUPPORTED;
2786 }
2787
2788 assert(new_size >= old_size && new_length >= old_length);
2789 return ACT_PRESERVING;
2790 }
2791
2792 // ----------------------------------------------------------------------
2793 // explicit template instantiations
2794 // ----------------------------------------------------------------------
2795
2796 template class Vector<NdbDictionary::Table*>;
2797 template class Vector<const NdbDictionary::Table*>;
2798 template class Vector<NdbDictionary::Tablespace*>;
2799 template class Vector<NdbDictionary::LogfileGroup*>;
2800 template class Vector<NdbDictionary::HashMap*>;
2801 template class Vector<NdbDictionary::Index*>;
2802 template class Vector<Vector<NdbDictionary::Index*> >;
2803
2804 // char array promotions/demotions
2805 template void * BackupRestore::convert_array< Hchar, Hchar >(const void *, void *, bool &);
2806 template void * BackupRestore::convert_array< Hchar, Hvarchar >(const void *, void *, bool &);
2807 template void * BackupRestore::convert_array< Hchar, Hlongvarchar >(const void *, void *, bool &);
2808 template void * BackupRestore::convert_array< Hvarchar, Hchar >(const void *, void *, bool &);
2809 template void * BackupRestore::convert_array< Hvarchar, Hvarchar >(const void *, void *, bool &);
2810 template void * BackupRestore::convert_array< Hvarchar, Hlongvarchar >(const void *, void *, bool &);
2811 template void * BackupRestore::convert_array< Hlongvarchar, Hchar >(const void *, void *, bool &);
2812 template void * BackupRestore::convert_array< Hlongvarchar, Hvarchar >(const void *, void *, bool &);
2813 template void * BackupRestore::convert_array< Hlongvarchar, Hlongvarchar >(const void *, void *, bool &);
2814
2815 // binary array promotions/demotions
2816 template void * BackupRestore::convert_array< Hbinary, Hbinary >(const void *, void *, bool &);
2817 template void * BackupRestore::convert_array< Hbinary, Hvarbinary >(const void *, void *, bool &);
2818 template void * BackupRestore::convert_array< Hbinary, Hlongvarbinary >(const void *, void *, bool &);
2819 template void * BackupRestore::convert_array< Hvarbinary, Hbinary >(const void *, void *, bool &);
2820 template void * BackupRestore::convert_array< Hvarbinary, Hvarbinary >(const void *, void *, bool &);
2821 template void * BackupRestore::convert_array< Hvarbinary, Hlongvarbinary >(const void *, void *, bool &);
2822 template void * BackupRestore::convert_array< Hlongvarbinary, Hbinary >(const void *, void *, bool &);
2823 template void * BackupRestore::convert_array< Hlongvarbinary, Hvarbinary >(const void *, void *, bool &);
2824 template void * BackupRestore::convert_array< Hlongvarbinary, Hlongvarbinary >(const void *, void *, bool &);
2825
2826 // integral promotions
2827 template void * BackupRestore::convert_integral<Hint8, Hint16>(const void *, void *, bool &);
2828 template void * BackupRestore::convert_integral<Hint8, Hint24>(const void *, void *, bool &);
2829 template void * BackupRestore::convert_integral<Hint8, Hint32>(const void *, void *, bool &);
2830 template void * BackupRestore::convert_integral<Hint8, Hint64>(const void *, void *, bool &);
2831 template void * BackupRestore::convert_integral<Hint16, Hint24>(const void *, void *, bool &);
2832 template void * BackupRestore::convert_integral<Hint16, Hint32>(const void *, void *, bool &);
2833 template void * BackupRestore::convert_integral<Hint16, Hint64>(const void *, void *, bool &);
2834 template void * BackupRestore::convert_integral<Hint24, Hint32>(const void *, void *, bool &);
2835 template void * BackupRestore::convert_integral<Hint24, Hint64>(const void *, void *, bool &);
2836 template void * BackupRestore::convert_integral<Hint32, Hint64>(const void *, void *, bool &);
2837 template void * BackupRestore::convert_integral<Huint8, Huint16>(const void *, void *, bool &);
2838 template void * BackupRestore::convert_integral<Huint8, Huint24>(const void *, void *, bool &);
2839 template void * BackupRestore::convert_integral<Huint8, Huint32>(const void *, void *, bool &);
2840 template void * BackupRestore::convert_integral<Huint8, Huint64>(const void *, void *, bool &);
2841 template void * BackupRestore::convert_integral<Huint16, Huint24>(const void *, void *, bool &);
2842 template void * BackupRestore::convert_integral<Huint16, Huint32>(const void *, void *, bool &);
2843 template void * BackupRestore::convert_integral<Huint16, Huint64>(const void *, void *, bool &);
2844 template void * BackupRestore::convert_integral<Huint24, Huint32>(const void *, void *, bool &);
2845 template void * BackupRestore::convert_integral<Huint24, Huint64>(const void *, void *, bool &);
2846 template void * BackupRestore::convert_integral<Huint32, Huint64>(const void *, void *, bool &);
2847
2848 // integral demotions
2849 template void * BackupRestore::convert_integral<Hint16, Hint8>(const void *, void *, bool &);
2850 template void * BackupRestore::convert_integral<Hint24, Hint8>(const void *, void *, bool &);
2851 template void * BackupRestore::convert_integral<Hint24, Hint16>(const void *, void *, bool &);
2852 template void * BackupRestore::convert_integral<Hint32, Hint8>(const void *, void *, bool &);
2853 template void * BackupRestore::convert_integral<Hint32, Hint16>(const void *, void *, bool &);
2854 template void * BackupRestore::convert_integral<Hint32, Hint24>(const void *, void *, bool &);
2855 template void * BackupRestore::convert_integral<Hint64, Hint8>(const void *, void *, bool &);
2856 template void * BackupRestore::convert_integral<Hint64, Hint16>(const void *, void *, bool &);
2857 template void * BackupRestore::convert_integral<Hint64, Hint24>(const void *, void *, bool &);
2858 template void * BackupRestore::convert_integral<Hint64, Hint32>(const void *, void *, bool &);
2859 template void * BackupRestore::convert_integral<Huint16, Huint8>(const void *, void *, bool &);
2860 template void * BackupRestore::convert_integral<Huint24, Huint8>(const void *, void *, bool &);
2861 template void * BackupRestore::convert_integral<Huint24, Huint16>(const void *, void *, bool &);
2862 template void * BackupRestore::convert_integral<Huint32, Huint8>(const void *, void *, bool &);
2863 template void * BackupRestore::convert_integral<Huint32, Huint16>(const void *, void *, bool &);
2864 template void * BackupRestore::convert_integral<Huint32, Huint24>(const void *, void *, bool &);
2865 template void * BackupRestore::convert_integral<Huint64, Huint8>(const void *, void *, bool &);
2866 template void * BackupRestore::convert_integral<Huint64, Huint16>(const void *, void *, bool &);
2867 template void * BackupRestore::convert_integral<Huint64, Huint24>(const void *, void *, bool &);
2868 template void * BackupRestore::convert_integral<Huint64, Huint32>(const void *, void *, bool &);
2869
2870 // integral signedness BackupRestore::conversions
2871 template void * BackupRestore::convert_integral<Hint8, Huint8>(const void *, void *, bool &);
2872 template void * BackupRestore::convert_integral<Hint16, Huint16>(const void *, void *, bool &);
2873 template void * BackupRestore::convert_integral<Hint24, Huint24>(const void *, void *, bool &);
2874 template void * BackupRestore::convert_integral<Hint32, Huint32>(const void *, void *, bool &);
2875 template void * BackupRestore::convert_integral<Hint64, Huint64>(const void *, void *, bool &);
2876 template void * BackupRestore::convert_integral<Huint8, Hint8>(const void *, void *, bool &);
2877 template void * BackupRestore::convert_integral<Huint16, Hint16>(const void *, void *, bool &);
2878 template void * BackupRestore::convert_integral<Huint24, Hint24>(const void *, void *, bool &);
2879 template void * BackupRestore::convert_integral<Huint32, Hint32>(const void *, void *, bool &);
2880 template void * BackupRestore::convert_integral<Huint64, Hint64>(const void *, void *, bool &);
2881
2882 // integral signedness+promotion BackupRestore::conversions
2883 template void * BackupRestore::convert_integral<Hint8, Huint16>(const void *, void *, bool &);
2884 template void * BackupRestore::convert_integral<Hint8, Huint24>(const void *, void *, bool &);
2885 template void * BackupRestore::convert_integral<Hint8, Huint32>(const void *, void *, bool &);
2886 template void * BackupRestore::convert_integral<Hint8, Huint64>(const void *, void *, bool &);
2887 template void * BackupRestore::convert_integral<Hint16, Huint24>(const void *, void *, bool &);
2888 template void * BackupRestore::convert_integral<Hint16, Huint32>(const void *, void *, bool &);
2889 template void * BackupRestore::convert_integral<Hint16, Huint64>(const void *, void *, bool &);
2890 template void * BackupRestore::convert_integral<Hint24, Huint32>(const void *, void *, bool &);
2891 template void * BackupRestore::convert_integral<Hint24, Huint64>(const void *, void *, bool &);
2892 template void * BackupRestore::convert_integral<Hint32, Huint64>(const void *, void *, bool &);
2893 template void * BackupRestore::convert_integral<Huint8, Hint16>(const void *, void *, bool &);
2894 template void * BackupRestore::convert_integral<Huint8, Hint24>(const void *, void *, bool &);
2895 template void * BackupRestore::convert_integral<Huint8, Hint32>(const void *, void *, bool &);
2896 template void * BackupRestore::convert_integral<Huint8, Hint64>(const void *, void *, bool &);
2897 template void * BackupRestore::convert_integral<Huint16, Hint24>(const void *, void *, bool &);
2898 template void * BackupRestore::convert_integral<Huint16, Hint32>(const void *, void *, bool &);
2899 template void * BackupRestore::convert_integral<Huint16, Hint64>(const void *, void *, bool &);
2900 template void * BackupRestore::convert_integral<Huint24, Hint32>(const void *, void *, bool &);
2901 template void * BackupRestore::convert_integral<Huint24, Hint64>(const void *, void *, bool &);
2902 template void * BackupRestore::convert_integral<Huint32, Hint64>(const void *, void *, bool &);
2903
2904 // integral signedness+demotion BackupRestore::conversions
2905 template void * BackupRestore::convert_integral<Hint16, Huint8>(const void *, void *, bool &);
2906 template void * BackupRestore::convert_integral<Hint24, Huint8>(const void *, void *, bool &);
2907 template void * BackupRestore::convert_integral<Hint24, Huint16>(const void *, void *, bool &);
2908 template void * BackupRestore::convert_integral<Hint32, Huint8>(const void *, void *, bool &);
2909 template void * BackupRestore::convert_integral<Hint32, Huint16>(const void *, void *, bool &);
2910 template void * BackupRestore::convert_integral<Hint32, Huint24>(const void *, void *, bool &);
2911 template void * BackupRestore::convert_integral<Hint64, Huint8>(const void *, void *, bool &);
2912 template void * BackupRestore::convert_integral<Hint64, Huint16>(const void *, void *, bool &);
2913 template void * BackupRestore::convert_integral<Hint64, Huint24>(const void *, void *, bool &);
2914 template void * BackupRestore::convert_integral<Hint64, Huint32>(const void *, void *, bool &);
2915 template void * BackupRestore::convert_integral<Huint16, Hint8>(const void *, void *, bool &);
2916 template void * BackupRestore::convert_integral<Huint24, Hint8>(const void *, void *, bool &);
2917 template void * BackupRestore::convert_integral<Huint24, Hint16>(const void *, void *, bool &);
2918 template void * BackupRestore::convert_integral<Huint32, Hint8>(const void *, void *, bool &);
2919 template void * BackupRestore::convert_integral<Huint32, Hint16>(const void *, void *, bool &);
2920 template void * BackupRestore::convert_integral<Huint32, Hint24>(const void *, void *, bool &);
2921 template void * BackupRestore::convert_integral<Huint64, Hint8>(const void *, void *, bool &);
2922 template void * BackupRestore::convert_integral<Huint64, Hint16>(const void *, void *, bool &);
2923 template void * BackupRestore::convert_integral<Huint64, Hint24>(const void *, void *, bool &);
2924 template void * BackupRestore::convert_integral<Huint64, Hint32>(const void *, void *, bool &);
2925