1 /*
2 Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License, version 2.0, for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 //****************************************************************************
26 //
27 // NAME
28 // Backup - Database backup / restore
29 //
30 //===========================================================================
31 #include "Backup.hpp"
32
33 #include <Properties.hpp>
34 #include <Configuration.hpp>
35
36 #include <EventLogger.hpp>
37 extern EventLogger * g_eventLogger;
38
39 #define JAM_FILE_ID 472
40
41
42 //extern const unsigned Ndbcntr::g_sysTableCount;
43
Backup(Block_context & ctx,Uint32 instanceNumber)44 Backup::Backup(Block_context& ctx, Uint32 instanceNumber) :
45 SimulatedBlock(BACKUP, ctx, instanceNumber),
46 c_nodes(c_nodePool),
47 c_backups(c_backupPool)
48 {
49 BLOCK_CONSTRUCTOR(Backup);
50
51 c_masterNodeId = getOwnNodeId();
52
53 // Add received signals
54 addRecSignal(GSN_READ_CONFIG_REQ, &Backup::execREAD_CONFIG_REQ);
55 addRecSignal(GSN_STTOR, &Backup::execSTTOR);
56 addRecSignal(GSN_DUMP_STATE_ORD, &Backup::execDUMP_STATE_ORD);
57 addRecSignal(GSN_READ_NODESCONF, &Backup::execREAD_NODESCONF);
58 addRecSignal(GSN_NODE_FAILREP, &Backup::execNODE_FAILREP);
59 addRecSignal(GSN_INCL_NODEREQ, &Backup::execINCL_NODEREQ);
60 addRecSignal(GSN_CONTINUEB, &Backup::execCONTINUEB);
61 addRecSignal(GSN_READ_CONFIG_REQ, &Backup::execREAD_CONFIG_REQ, true);
62
63 addRecSignal(GSN_SCAN_HBREP, &Backup::execSCAN_HBREP);
64 addRecSignal(GSN_TRANSID_AI, &Backup::execTRANSID_AI);
65 addRecSignal(GSN_SCAN_FRAGREF, &Backup::execSCAN_FRAGREF);
66 addRecSignal(GSN_SCAN_FRAGCONF, &Backup::execSCAN_FRAGCONF);
67
68 addRecSignal(GSN_BACKUP_TRIG_REQ, &Backup::execBACKUP_TRIG_REQ);
69 addRecSignal(GSN_TRIG_ATTRINFO, &Backup::execTRIG_ATTRINFO);
70 addRecSignal(GSN_FIRE_TRIG_ORD, &Backup::execFIRE_TRIG_ORD);
71
72 addRecSignal(GSN_LIST_TABLES_CONF, &Backup::execLIST_TABLES_CONF);
73 addRecSignal(GSN_GET_TABINFOREF, &Backup::execGET_TABINFOREF);
74 addRecSignal(GSN_GET_TABINFO_CONF, &Backup::execGET_TABINFO_CONF);
75
76 addRecSignal(GSN_CREATE_TRIG_IMPL_REF, &Backup::execCREATE_TRIG_IMPL_REF);
77 addRecSignal(GSN_CREATE_TRIG_IMPL_CONF, &Backup::execCREATE_TRIG_IMPL_CONF);
78
79 addRecSignal(GSN_DROP_TRIG_IMPL_REF, &Backup::execDROP_TRIG_IMPL_REF);
80 addRecSignal(GSN_DROP_TRIG_IMPL_CONF, &Backup::execDROP_TRIG_IMPL_CONF);
81
82 addRecSignal(GSN_DIH_SCAN_TAB_CONF, &Backup::execDIH_SCAN_TAB_CONF);
83 addRecSignal(GSN_DIH_SCAN_GET_NODES_CONF,
84 &Backup::execDIH_SCAN_GET_NODES_CONF);
85
86 addRecSignal(GSN_FSOPENREF, &Backup::execFSOPENREF, true);
87 addRecSignal(GSN_FSOPENCONF, &Backup::execFSOPENCONF);
88
89 addRecSignal(GSN_FSCLOSEREF, &Backup::execFSCLOSEREF, true);
90 addRecSignal(GSN_FSCLOSECONF, &Backup::execFSCLOSECONF);
91
92 addRecSignal(GSN_FSAPPENDREF, &Backup::execFSAPPENDREF, true);
93 addRecSignal(GSN_FSAPPENDCONF, &Backup::execFSAPPENDCONF);
94
95 addRecSignal(GSN_FSREMOVEREF, &Backup::execFSREMOVEREF, true);
96 addRecSignal(GSN_FSREMOVECONF, &Backup::execFSREMOVECONF);
97
98 /*****/
99 addRecSignal(GSN_BACKUP_REQ, &Backup::execBACKUP_REQ);
100 addRecSignal(GSN_ABORT_BACKUP_ORD, &Backup::execABORT_BACKUP_ORD);
101
102 addRecSignal(GSN_DEFINE_BACKUP_REQ, &Backup::execDEFINE_BACKUP_REQ);
103 addRecSignal(GSN_DEFINE_BACKUP_REF, &Backup::execDEFINE_BACKUP_REF);
104 addRecSignal(GSN_DEFINE_BACKUP_CONF, &Backup::execDEFINE_BACKUP_CONF);
105
106 addRecSignal(GSN_START_BACKUP_REQ, &Backup::execSTART_BACKUP_REQ);
107 addRecSignal(GSN_START_BACKUP_REF, &Backup::execSTART_BACKUP_REF);
108 addRecSignal(GSN_START_BACKUP_CONF, &Backup::execSTART_BACKUP_CONF);
109
110 addRecSignal(GSN_BACKUP_FRAGMENT_REQ, &Backup::execBACKUP_FRAGMENT_REQ);
111 addRecSignal(GSN_BACKUP_FRAGMENT_REF, &Backup::execBACKUP_FRAGMENT_REF);
112 addRecSignal(GSN_BACKUP_FRAGMENT_CONF, &Backup::execBACKUP_FRAGMENT_CONF);
113
114 addRecSignal(GSN_BACKUP_FRAGMENT_COMPLETE_REP,
115 &Backup::execBACKUP_FRAGMENT_COMPLETE_REP);
116
117 addRecSignal(GSN_STOP_BACKUP_REQ, &Backup::execSTOP_BACKUP_REQ);
118 addRecSignal(GSN_STOP_BACKUP_REF, &Backup::execSTOP_BACKUP_REF);
119 addRecSignal(GSN_STOP_BACKUP_CONF, &Backup::execSTOP_BACKUP_CONF);
120
121 //addRecSignal(GSN_BACKUP_STATUS_REQ, &Backup::execBACKUP_STATUS_REQ);
122 //addRecSignal(GSN_BACKUP_STATUS_CONF, &Backup::execBACKUP_STATUS_CONF);
123
124 addRecSignal(GSN_UTIL_SEQUENCE_REF, &Backup::execUTIL_SEQUENCE_REF);
125 addRecSignal(GSN_UTIL_SEQUENCE_CONF, &Backup::execUTIL_SEQUENCE_CONF);
126
127 addRecSignal(GSN_WAIT_GCP_REF, &Backup::execWAIT_GCP_REF);
128 addRecSignal(GSN_WAIT_GCP_CONF, &Backup::execWAIT_GCP_CONF);
129 addRecSignal(GSN_BACKUP_LOCK_TAB_CONF, &Backup::execBACKUP_LOCK_TAB_CONF);
130 addRecSignal(GSN_BACKUP_LOCK_TAB_REF, &Backup::execBACKUP_LOCK_TAB_REF);
131
132 addRecSignal(GSN_LCP_STATUS_REQ, &Backup::execLCP_STATUS_REQ);
133
134 /**
135 * Testing
136 */
137 addRecSignal(GSN_BACKUP_REF, &Backup::execBACKUP_REF);
138 addRecSignal(GSN_BACKUP_CONF, &Backup::execBACKUP_CONF);
139 addRecSignal(GSN_BACKUP_ABORT_REP, &Backup::execBACKUP_ABORT_REP);
140 addRecSignal(GSN_BACKUP_COMPLETE_REP, &Backup::execBACKUP_COMPLETE_REP);
141
142 addRecSignal(GSN_LCP_PREPARE_REQ, &Backup::execLCP_PREPARE_REQ);
143 addRecSignal(GSN_END_LCPREQ, &Backup::execEND_LCPREQ);
144
145 addRecSignal(GSN_DBINFO_SCANREQ, &Backup::execDBINFO_SCANREQ);
146
147 addRecSignal(GSN_CHECK_NODE_RESTARTCONF,
148 &Backup::execCHECK_NODE_RESTARTCONF);
149 }
150
~Backup()151 Backup::~Backup()
152 {
153 }
154
155 BLOCK_FUNCTIONS(Backup)
156
157 template class ArrayPool<Backup::Page32>;
158 template class ArrayPool<Backup::Fragment>;
159
160 void
execREAD_CONFIG_REQ(Signal * signal)161 Backup::execREAD_CONFIG_REQ(Signal* signal)
162 {
163 const ReadConfigReq * req = (ReadConfigReq*)signal->getDataPtr();
164 Uint32 ref = req->senderRef;
165 Uint32 senderData = req->senderData;
166 ndbrequire(req->noOfParameters == 0);
167 jamEntry();
168
169 const ndb_mgm_configuration_iterator * p =
170 m_ctx.m_config.getOwnConfigIterator();
171 ndbrequire(p != 0);
172
173 c_defaults.m_disk_write_speed_min = 10 * (1024 * 1024);
174 c_defaults.m_disk_write_speed_max = 20 * (1024 * 1024);
175 c_defaults.m_disk_write_speed_max_other_node_restart = 50 * (1024 * 1024);
176 c_defaults.m_disk_write_speed_max_own_restart = 100 * (1024 * 1024);
177 c_defaults.m_disk_synch_size = 4 * (1024 * 1024);
178 c_defaults.m_o_direct = true;
179
180 Uint32 noBackups = 0, noTables = 0, noAttribs = 0, noFrags = 0;
181 ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_DB_DISCLESS,
182 &c_defaults.m_diskless));
183 ndb_mgm_get_int_parameter(p, CFG_DB_O_DIRECT,
184 &c_defaults.m_o_direct);
185
186 ndb_mgm_get_int64_parameter(p, CFG_DB_MIN_DISK_WRITE_SPEED,
187 &c_defaults.m_disk_write_speed_min);
188 ndb_mgm_get_int64_parameter(p, CFG_DB_MAX_DISK_WRITE_SPEED,
189 &c_defaults.m_disk_write_speed_max);
190 ndb_mgm_get_int64_parameter(p,
191 CFG_DB_MAX_DISK_WRITE_SPEED_OTHER_NODE_RESTART,
192 &c_defaults.m_disk_write_speed_max_other_node_restart);
193 ndb_mgm_get_int64_parameter(p,
194 CFG_DB_MAX_DISK_WRITE_SPEED_OWN_RESTART,
195 &c_defaults.m_disk_write_speed_max_own_restart);
196
197 ndb_mgm_get_int_parameter(p, CFG_DB_DISK_SYNCH_SIZE,
198 &c_defaults.m_disk_synch_size);
199 ndb_mgm_get_int_parameter(p, CFG_DB_COMPRESSED_BACKUP,
200 &c_defaults.m_compressed_backup);
201 ndb_mgm_get_int_parameter(p, CFG_DB_COMPRESSED_LCP,
202 &c_defaults.m_compressed_lcp);
203
204 calculate_real_disk_write_speed_parameters();
205
206 jam();
207 m_backup_report_frequency = 0;
208 ndb_mgm_get_int_parameter(p, CFG_DB_BACKUP_REPORT_FREQUENCY,
209 &m_backup_report_frequency);
210
211 ndb_mgm_get_int_parameter(p, CFG_DB_PARALLEL_BACKUPS, &noBackups);
212 // ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_DB_NO_TABLES, &noTables));
213 ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_DICT_TABLE, &noTables));
214 ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_DB_NO_ATTRIBUTES, &noAttribs));
215 ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_DIH_FRAG_CONNECT, &noFrags));
216
217 noAttribs++; //RT 527 bug fix
218
219 c_nodePool.setSize(MAX_NDB_NODES);
220 c_backupPool.setSize(noBackups + 1);
221 c_backupFilePool.setSize(3 * noBackups + 1);
222 c_tablePool.setSize(noBackups * noTables + 1);
223 c_triggerPool.setSize(noBackups * 3 * noTables);
224 c_fragmentPool.setSize(noBackups * noFrags + 1);
225
226 jam();
227
228 const Uint32 DEFAULT_WRITE_SIZE = (256 * 1024);
229 const Uint32 DEFAULT_MAX_WRITE_SIZE = (1024 * 1024);
230 const Uint32 DEFAULT_BUFFER_SIZE = (16 * 1024 * 1024);
231
232 Uint32 szDataBuf = DEFAULT_BUFFER_SIZE;
233 Uint32 szLogBuf = DEFAULT_BUFFER_SIZE;
234 Uint32 szWrite = DEFAULT_WRITE_SIZE;
235 Uint32 maxWriteSize = DEFAULT_MAX_WRITE_SIZE;
236
237 ndb_mgm_get_int_parameter(p, CFG_DB_BACKUP_DATA_BUFFER_MEM, &szDataBuf);
238 ndb_mgm_get_int_parameter(p, CFG_DB_BACKUP_LOG_BUFFER_MEM, &szLogBuf);
239 ndb_mgm_get_int_parameter(p, CFG_DB_BACKUP_WRITE_SIZE, &szWrite);
240 ndb_mgm_get_int_parameter(p, CFG_DB_BACKUP_MAX_WRITE_SIZE, &maxWriteSize);
241
242 if (maxWriteSize < szWrite)
243 {
244 /**
245 * max can't be lower than min
246 */
247 maxWriteSize = szWrite;
248 }
249 if ((maxWriteSize % szWrite) != 0)
250 {
251 /**
252 * max needs to be a multiple of min
253 */
254 maxWriteSize = (maxWriteSize + szWrite - 1) / szWrite;
255 maxWriteSize *= szWrite;
256 }
257
258 /**
259 * add min writesize to buffer size...and the alignment added here and there
260 */
261 Uint32 extra = szWrite + 4 * (/* align * 512b */ 128);
262
263 szDataBuf += extra;
264 szLogBuf += extra;
265
266 c_defaults.m_logBufferSize = szLogBuf;
267 c_defaults.m_dataBufferSize = szDataBuf;
268 c_defaults.m_minWriteSize = szWrite;
269 c_defaults.m_maxWriteSize = maxWriteSize;
270 c_defaults.m_lcp_buffer_size = szDataBuf;
271
272 /* We deprecate the use of BackupMemory, it serves no purpose at all. */
273 Uint32 szMem = 0;
274 ndb_mgm_get_int_parameter(p, CFG_DB_BACKUP_MEM, &szMem);
275
276 if (szMem != (32 * 1024 * 1024))
277 {
278 jam();
279 g_eventLogger->info("BackupMemory parameter setting ignored,"
280 " BackupMemory deprecated");
281 }
282 szMem = szDataBuf + szLogBuf;
283
284 /**
285 * We allocate szDataBuf + szLogBuf pages for Backups and
286 * szDataBuf pages for LCPs.
287 */
288 Uint32 noPages =
289 (szMem + sizeof(Page32) - 1) / sizeof(Page32) +
290 (c_defaults.m_lcp_buffer_size + sizeof(Page32) - 1) / sizeof(Page32);
291
292 // We need to allocate an additional of 2 pages. 1 page because of a bug in
293 // ArrayPool and another one for DICTTABINFO.
294 c_pagePool.setSize(noPages + NO_OF_PAGES_META_FILE + 2, true);
295
296 jam();
297
298 { // Init all tables
299 SLList<Table> tables(c_tablePool);
300 TablePtr ptr;
301 while (tables.seizeFirst(ptr)){
302 new (ptr.p) Table(c_fragmentPool);
303 }
304 jam();
305 while (tables.releaseFirst())
306 {
307 ;
308 }
309 jam();
310 }
311
312 {
313 SLList<BackupFile> ops(c_backupFilePool);
314 BackupFilePtr ptr;
315 while (ops.seizeFirst(ptr)){
316 new (ptr.p) BackupFile(* this, c_pagePool);
317 }
318 jam();
319 while (ops.releaseFirst())
320 {
321 ;
322 }
323 jam();
324 }
325
326 {
327 SLList<BackupRecord> recs(c_backupPool);
328 BackupRecordPtr ptr;
329 while (recs.seizeFirst(ptr)){
330 new (ptr.p) BackupRecord(* this, c_tablePool,
331 c_backupFilePool, c_triggerPool);
332 }
333 jam();
334 while (recs.releaseFirst())
335 {
336 ;
337 }
338 jam();
339 }
340
341 // Initialize BAT for interface to file system
342 {
343 Page32Ptr p;
344 ndbrequire(c_pagePool.seizeId(p, 0));
345 c_startOfPages = (Uint32 *)p.p;
346 c_pagePool.release(p);
347
348 NewVARIABLE* bat = allocateBat(1);
349 bat[0].WA = c_startOfPages;
350 bat[0].nrr = c_pagePool.getSize()*sizeof(Page32)/sizeof(Uint32);
351 }
352
353 ReadConfigConf * conf = (ReadConfigConf*)signal->getDataPtrSend();
354 conf->senderRef = reference();
355 conf->senderData = senderData;
356 sendSignal(ref, GSN_READ_CONFIG_CONF, signal,
357 ReadConfigConf::SignalLength, JBB);
358 }
359
360 /* Broken out in its own routine to enable setting via DUMP command. */
calculate_real_disk_write_speed_parameters(void)361 void Backup::calculate_real_disk_write_speed_parameters(void)
362 {
363 if (c_defaults.m_disk_write_speed_max < c_defaults.m_disk_write_speed_min)
364 {
365 /**
366 * By setting max disk write speed equal or smaller than the minimum
367 * we will remove the adaptiveness of the LCP speed.
368 */
369 jam();
370 ndbout << "Setting MaxDiskWriteSpeed to MinDiskWriteSpeed since max < min"
371 << endl;
372 c_defaults.m_disk_write_speed_max = c_defaults.m_disk_write_speed_min;
373 }
374
375 if (c_defaults.m_disk_write_speed_max_other_node_restart <
376 c_defaults.m_disk_write_speed_max)
377 {
378 /**
379 * By setting max disk write speed during restart equal or smaller than
380 * the maximum we will remove the extra adaptiveness of the LCP speed
381 * at other nodes restarts.
382 */
383 jam();
384 ndbout << "MaxDiskWriteSpeed larger than MaxDiskWriteSpeedOtherNodeRestart"
385 << " setting both to MaxDiskWriteSpeed" << endl;
386 c_defaults.m_disk_write_speed_max_other_node_restart =
387 c_defaults.m_disk_write_speed_max;
388 }
389
390 if (c_defaults.m_disk_write_speed_max_own_restart <
391 c_defaults.m_disk_write_speed_max_other_node_restart)
392 {
393 /**
394 * By setting restart disk write speed during our restart equal or
395 * smaller than the maximum we will remove the extra adaptiveness of the
396 * LCP speed at other nodes restarts.
397 */
398 jam();
399 ndbout << "Setting MaxDiskWriteSpeedOwnRestart to "
400 << " MaxDiskWriteSpeedOtherNodeRestart since it was smaller"
401 << endl;
402 c_defaults.m_disk_write_speed_max_own_restart =
403 c_defaults.m_disk_write_speed_max_other_node_restart;
404 }
405
406 /*
407 We adjust the disk speed parameters from bytes per second to rather be
408 words per 100 milliseconds. We convert disk synch size from bytes per
409 second to words per second.
410 */
411 c_defaults.m_disk_write_speed_min /=
412 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
413 c_defaults.m_disk_write_speed_max /=
414 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
415 c_defaults.m_disk_write_speed_max_other_node_restart /=
416 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
417 c_defaults.m_disk_write_speed_max_own_restart /=
418 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
419
420 Uint32 num_ldm_threads = globalData.ndbMtLqhThreads;
421 if (num_ldm_threads == 0)
422 {
423 /* We are running with ndbd binary */
424 jam();
425 num_ldm_threads = 1;
426 }
427 c_defaults.m_disk_write_speed_min /= num_ldm_threads;
428 c_defaults.m_disk_write_speed_max /= num_ldm_threads;
429 c_defaults.m_disk_write_speed_max_other_node_restart /= num_ldm_threads;
430 c_defaults.m_disk_write_speed_max_own_restart /= num_ldm_threads;
431 }
432
restore_disk_write_speed_numbers(void)433 void Backup::restore_disk_write_speed_numbers(void)
434 {
435 c_defaults.m_disk_write_speed_min *=
436 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
437 c_defaults.m_disk_write_speed_max *=
438 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
439 c_defaults.m_disk_write_speed_max_other_node_restart *=
440 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
441 c_defaults.m_disk_write_speed_max_own_restart *=
442 CURR_DISK_SPEED_CONVERSION_FACTOR_TO_SECONDS;
443
444 Uint32 num_ldm_threads = globalData.ndbMtLqhThreads;
445 if (num_ldm_threads == 0)
446 {
447 /* We are running with ndbd binary */
448 jam();
449 num_ldm_threads = 1;
450 }
451
452 c_defaults.m_disk_write_speed_min *= num_ldm_threads;
453 c_defaults.m_disk_write_speed_max *= num_ldm_threads;
454 c_defaults.m_disk_write_speed_max_other_node_restart *= num_ldm_threads;
455 c_defaults.m_disk_write_speed_max_own_restart *= num_ldm_threads;
456 }
457