1 /* Copyright (c) 2003-2008 MySQL AB
2 Use is subject to license terms
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; version 2 of the License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 GNU General Public License for more details.
12
13 You should have received a copy of the GNU General Public License
14 along with this program; if not, write to the Free Software
15 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */
16
17 #include <NDBT.hpp>
18 #include <NDBT_Test.hpp>
19 #include <HugoTransactions.hpp>
20 #include <UtilTransactions.hpp>
21 #include <NdbRestarter.hpp>
22 #include <NdbRestarts.hpp>
23 #include <Vector.hpp>
24 #include <signaldata/DumpStateOrd.hpp>
25 #include <Bitmask.hpp>
26 #include <RefConvert.hpp>
27 #include <NdbEnv.h>
28
runLoadTable(NDBT_Context * ctx,NDBT_Step * step)29 int runLoadTable(NDBT_Context* ctx, NDBT_Step* step){
30
31 int records = ctx->getNumRecords();
32 HugoTransactions hugoTrans(*ctx->getTab());
33 if (hugoTrans.loadTable(GETNDB(step), records) != 0){
34 return NDBT_FAILED;
35 }
36 return NDBT_OK;
37 }
38
runFillTable(NDBT_Context * ctx,NDBT_Step * step)39 int runFillTable(NDBT_Context* ctx, NDBT_Step* step){
40
41 HugoTransactions hugoTrans(*ctx->getTab());
42 if (hugoTrans.fillTable(GETNDB(step)) != 0){
43 return NDBT_FAILED;
44 }
45 return NDBT_OK;
46 }
47
runInsertUntilStopped(NDBT_Context * ctx,NDBT_Step * step)48 int runInsertUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
49 int result = NDBT_OK;
50 int records = ctx->getNumRecords();
51 int i = 0;
52 HugoTransactions hugoTrans(*ctx->getTab());
53 while (ctx->isTestStopped() == false) {
54 g_info << i << ": ";
55 if (hugoTrans.loadTable(GETNDB(step), records) != 0){
56 return NDBT_FAILED;
57 }
58 i++;
59 }
60 return result;
61 }
62
runClearTable(NDBT_Context * ctx,NDBT_Step * step)63 int runClearTable(NDBT_Context* ctx, NDBT_Step* step){
64 int records = ctx->getNumRecords();
65
66 UtilTransactions utilTrans(*ctx->getTab());
67 if (utilTrans.clearTable(GETNDB(step), records) != 0){
68 return NDBT_FAILED;
69 }
70 return NDBT_OK;
71 }
72
runClearTableUntilStopped(NDBT_Context * ctx,NDBT_Step * step)73 int runClearTableUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
74 int records = ctx->getNumRecords();
75 int i = 0;
76
77 UtilTransactions utilTrans(*ctx->getTab());
78 while (ctx->isTestStopped() == false) {
79 g_info << i << ": ";
80 if (utilTrans.clearTable(GETNDB(step), records) != 0){
81 return NDBT_FAILED;
82 }
83 i++;
84 }
85 return NDBT_OK;
86 }
87
runScanReadUntilStopped(NDBT_Context * ctx,NDBT_Step * step)88 int runScanReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
89 int result = NDBT_OK;
90 int records = ctx->getNumRecords();
91 int i = 0;
92 HugoTransactions hugoTrans(*ctx->getTab());
93 while (ctx->isTestStopped() == false) {
94 g_info << i << ": ";
95 if (hugoTrans.scanReadRecords(GETNDB(step), records) != 0){
96 return NDBT_FAILED;
97 }
98 i++;
99 }
100 return result;
101 }
102
runPkReadUntilStopped(NDBT_Context * ctx,NDBT_Step * step)103 int runPkReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
104 int result = NDBT_OK;
105 int records = ctx->getNumRecords();
106 NdbOperation::LockMode lm =
107 (NdbOperation::LockMode)ctx->getProperty("ReadLockMode",
108 (Uint32)NdbOperation::LM_Read);
109 int i = 0;
110 HugoTransactions hugoTrans(*ctx->getTab());
111 while (ctx->isTestStopped() == false) {
112 g_info << i << ": ";
113 int rows = (rand()%records)+1;
114 int batch = (rand()%rows)+1;
115 if (hugoTrans.pkReadRecords(GETNDB(step), rows, batch, lm) != 0){
116 return NDBT_FAILED;
117 }
118 i++;
119 }
120 return result;
121 }
122
runPkUpdateUntilStopped(NDBT_Context * ctx,NDBT_Step * step)123 int runPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
124 int result = NDBT_OK;
125 int records = ctx->getNumRecords();
126 int multiop = ctx->getProperty("MULTI_OP", 1);
127 Ndb* pNdb = GETNDB(step);
128 int i = 0;
129
130 HugoOperations hugoOps(*ctx->getTab());
131 while (ctx->isTestStopped() == false)
132 {
133 g_info << i << ": ";
134 int batch = (rand()%records)+1;
135 int row = rand() % records;
136
137 if (batch > 25)
138 batch = 25;
139
140 if(row + batch > records)
141 batch = records - row;
142
143 if(hugoOps.startTransaction(pNdb) != 0)
144 goto err;
145
146 if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
147 goto err;
148
149 for (int j = 1; j<multiop; j++)
150 {
151 if(hugoOps.execute_NoCommit(pNdb) != 0)
152 goto err;
153
154 if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
155 goto err;
156 }
157
158 if(hugoOps.execute_Commit(pNdb) != 0)
159 goto err;
160
161 hugoOps.closeTransaction(pNdb);
162
163 continue;
164
165 err:
166 NdbConnection* pCon = hugoOps.getTransaction();
167 if(pCon == 0)
168 continue;
169 NdbError error = pCon->getNdbError();
170 hugoOps.closeTransaction(pNdb);
171 if (error.status == NdbError::TemporaryError){
172 NdbSleep_MilliSleep(50);
173 continue;
174 }
175 return NDBT_FAILED;
176
177 i++;
178 }
179 return result;
180 }
181
runPkReadPkUpdateUntilStopped(NDBT_Context * ctx,NDBT_Step * step)182 int runPkReadPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
183 int result = NDBT_OK;
184 int records = ctx->getNumRecords();
185 Ndb* pNdb = GETNDB(step);
186 int i = 0;
187 HugoOperations hugoOps(*ctx->getTab());
188 while (ctx->isTestStopped() == false) {
189 g_info << i++ << ": ";
190 int rows = (rand()%records)+1;
191 int batch = (rand()%rows)+1;
192 int row = (records - rows) ? rand() % (records - rows) : 0;
193
194 int j,k;
195 for(j = 0; j<rows; j += batch)
196 {
197 k = batch;
198 if(j+k > rows)
199 k = rows - j;
200
201 if(hugoOps.startTransaction(pNdb) != 0)
202 goto err;
203
204 if(hugoOps.pkReadRecord(pNdb, row+j, k, NdbOperation::LM_Exclusive) != 0)
205 goto err;
206
207 if(hugoOps.execute_NoCommit(pNdb) != 0)
208 goto err;
209
210 if(hugoOps.pkUpdateRecord(pNdb, row+j, k, rand()) != 0)
211 goto err;
212
213 if(hugoOps.execute_Commit(pNdb) != 0)
214 goto err;
215
216 if(hugoOps.closeTransaction(pNdb) != 0)
217 return NDBT_FAILED;
218 }
219
220 continue;
221 err:
222 NdbConnection* pCon = hugoOps.getTransaction();
223 if(pCon == 0)
224 continue;
225 NdbError error = pCon->getNdbError();
226 hugoOps.closeTransaction(pNdb);
227 if (error.status == NdbError::TemporaryError){
228 NdbSleep_MilliSleep(50);
229 continue;
230 }
231 return NDBT_FAILED;
232 }
233 return NDBT_OK;
234 }
235
runScanUpdateUntilStopped(NDBT_Context * ctx,NDBT_Step * step)236 int runScanUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
237 int result = NDBT_OK;
238 int records = ctx->getNumRecords();
239 int parallelism = ctx->getProperty("Parallelism", 1);
240 int abort = ctx->getProperty("AbortProb", (Uint32)0);
241 int i = 0;
242 HugoTransactions hugoTrans(*ctx->getTab());
243 while (ctx->isTestStopped() == false) {
244 g_info << i << ": ";
245 if (hugoTrans.scanUpdateRecords(GETNDB(step), records, abort,
246 parallelism) == NDBT_FAILED){
247 return NDBT_FAILED;
248 }
249 i++;
250 }
251 return result;
252 }
253
runScanReadVerify(NDBT_Context * ctx,NDBT_Step * step)254 int runScanReadVerify(NDBT_Context* ctx, NDBT_Step* step){
255 int records = ctx->getNumRecords();
256 HugoTransactions hugoTrans(*ctx->getTab());
257
258 if (hugoTrans.scanReadRecords(GETNDB(step), records, 0, 64) != 0){
259 return NDBT_FAILED;
260 }
261 return NDBT_OK;
262 }
263
runRestarter(NDBT_Context * ctx,NDBT_Step * step)264 int runRestarter(NDBT_Context* ctx, NDBT_Step* step){
265 int result = NDBT_OK;
266 int loops = ctx->getNumLoops();
267 int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0);
268 NdbRestarter restarter;
269 int i = 0;
270 int lastId = 0;
271
272 if (restarter.getNumDbNodes() < 2){
273 ctx->stopTest();
274 return NDBT_OK;
275 }
276
277 if(restarter.waitClusterStarted() != 0){
278 g_err << "Cluster failed to start" << endl;
279 return NDBT_FAILED;
280 }
281
282 loops *= restarter.getNumDbNodes();
283 while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
284
285 int id = lastId % restarter.getNumDbNodes();
286 int nodeId = restarter.getDbNodeId(id);
287 ndbout << "Restart node " << nodeId << endl;
288 if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){
289 g_err << "Failed to restartNextDbNode" << endl;
290 result = NDBT_FAILED;
291 break;
292 }
293
294 if (restarter.waitNodesNoStart(&nodeId, 1))
295 {
296 g_err << "Failed to waitNodesNoStart" << endl;
297 result = NDBT_FAILED;
298 break;
299 }
300
301 if (restarter.startNodes(&nodeId, 1))
302 {
303 g_err << "Failed to start node" << endl;
304 result = NDBT_FAILED;
305 break;
306 }
307
308 if(restarter.waitClusterStarted() != 0){
309 g_err << "Cluster failed to start" << endl;
310 result = NDBT_FAILED;
311 break;
312 }
313
314 ctx->sync_up_and_wait("PauseThreads", sync_threads);
315
316 lastId++;
317 i++;
318 }
319
320 ctx->stopTest();
321
322 return result;
323 }
324
runCheckAllNodesStarted(NDBT_Context * ctx,NDBT_Step * step)325 int runCheckAllNodesStarted(NDBT_Context* ctx, NDBT_Step* step){
326 NdbRestarter restarter;
327
328 if(restarter.waitClusterStarted(1) != 0){
329 g_err << "All nodes was not started " << endl;
330 return NDBT_FAILED;
331 }
332
333 return NDBT_OK;
334 }
335
336
337
runRestarts(NDBT_Context * ctx,NDBT_Step * step)338 int runRestarts(NDBT_Context* ctx, NDBT_Step* step){
339 int result = NDBT_OK;
340 int loops = ctx->getNumLoops();
341 NDBT_TestCase* pCase = ctx->getCase();
342 NdbRestarts restarts;
343 int i = 0;
344 int timeout = 240;
345
346 while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
347
348 if(restarts.executeRestart(pCase->getName(), timeout) != 0){
349 g_err << "Failed to executeRestart(" <<pCase->getName() <<")" << endl;
350 result = NDBT_FAILED;
351 break;
352 }
353 i++;
354 }
355 ctx->stopTest();
356 return result;
357 }
358
runDirtyRead(NDBT_Context * ctx,NDBT_Step * step)359 int runDirtyRead(NDBT_Context* ctx, NDBT_Step* step){
360 int result = NDBT_OK;
361 int loops = ctx->getNumLoops();
362 int records = ctx->getNumRecords();
363 NdbRestarter restarter;
364 HugoOperations hugoOps(*ctx->getTab());
365 Ndb* pNdb = GETNDB(step);
366
367 int i = 0;
368 while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
369 g_info << i << ": ";
370
371 int id = i % restarter.getNumDbNodes();
372 int nodeId = restarter.getDbNodeId(id);
373 ndbout << "Restart node " << nodeId << endl;
374 restarter.insertErrorInNode(nodeId, 5041);
375 restarter.insertErrorInAllNodes(8048 + (i & 1));
376
377 for(int j = 0; j<records; j++){
378 if(hugoOps.startTransaction(pNdb) != 0)
379 return NDBT_FAILED;
380
381 if(hugoOps.pkReadRecord(pNdb, j, 1, NdbOperation::LM_CommittedRead) != 0)
382 goto err;
383
384 int res;
385 if((res = hugoOps.execute_Commit(pNdb)) == 4119)
386 goto done;
387
388 if(res != 0)
389 goto err;
390
391 if(hugoOps.closeTransaction(pNdb) != 0)
392 return NDBT_FAILED;
393 }
394 done:
395 if(hugoOps.closeTransaction(pNdb) != 0)
396 return NDBT_FAILED;
397
398 i++;
399 restarter.waitClusterStarted(60) ;
400 }
401 return result;
402 err:
403 hugoOps.closeTransaction(pNdb);
404 return NDBT_FAILED;
405 }
406
runLateCommit(NDBT_Context * ctx,NDBT_Step * step)407 int runLateCommit(NDBT_Context* ctx, NDBT_Step* step){
408 int result = NDBT_OK;
409 int loops = ctx->getNumLoops();
410 int records = ctx->getNumRecords();
411 NdbRestarter restarter;
412 HugoOperations hugoOps(*ctx->getTab());
413 Ndb* pNdb = GETNDB(step);
414
415 int i = 0;
416 while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
417 g_info << i << ": ";
418
419 if(hugoOps.startTransaction(pNdb) != 0)
420 return NDBT_FAILED;
421
422 if(hugoOps.pkUpdateRecord(pNdb, 1, 128) != 0)
423 return NDBT_FAILED;
424
425 if(hugoOps.execute_NoCommit(pNdb) != 0)
426 return NDBT_FAILED;
427
428 Uint32 transNode= hugoOps.getTransaction()->getConnectedNodeId();
429 int id = i % restarter.getNumDbNodes();
430 int nodeId;
431 while((nodeId = restarter.getDbNodeId(id)) == transNode)
432 id = (id + 1) % restarter.getNumDbNodes();
433
434 ndbout << "Restart node " << nodeId << endl;
435
436 restarter.restartOneDbNode(nodeId,
437 /** initial */ false,
438 /** nostart */ true,
439 /** abort */ true);
440
441 restarter.waitNodesNoStart(&nodeId, 1);
442
443 int res;
444 if(i & 1)
445 res= hugoOps.execute_Commit(pNdb);
446 else
447 res= hugoOps.execute_Rollback(pNdb);
448
449 ndbout_c("res= %d", res);
450
451 hugoOps.closeTransaction(pNdb);
452
453 restarter.startNodes(&nodeId, 1);
454 restarter.waitNodesStarted(&nodeId, 1);
455
456 if(i & 1)
457 {
458 if(res != 286)
459 return NDBT_FAILED;
460 }
461 else
462 {
463 if(res != 0)
464 return NDBT_FAILED;
465 }
466 i++;
467 }
468
469 return NDBT_OK;
470 }
471
runBug15587(NDBT_Context * ctx,NDBT_Step * step)472 int runBug15587(NDBT_Context* ctx, NDBT_Step* step){
473 int result = NDBT_OK;
474 int loops = ctx->getNumLoops();
475 int records = ctx->getNumRecords();
476 NdbRestarter restarter;
477
478 Uint32 tableId = ctx->getTab()->getTableId();
479 int dump[2] = { DumpStateOrd::LqhErrorInsert5042, 0 };
480 dump[1] = tableId;
481
482 int nodeId = restarter.getDbNodeId(1);
483
484 ndbout << "Restart node " << nodeId << endl;
485
486 if (restarter.restartOneDbNode(nodeId,
487 /** initial */ false,
488 /** nostart */ true,
489 /** abort */ true))
490 return NDBT_FAILED;
491
492 if (restarter.waitNodesNoStart(&nodeId, 1))
493 return NDBT_FAILED;
494
495 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
496
497 if (restarter.dumpStateOneNode(nodeId, val2, 2))
498 return NDBT_FAILED;
499
500 if (restarter.dumpStateOneNode(nodeId, dump, 2))
501 return NDBT_FAILED;
502
503 if (restarter.startNodes(&nodeId, 1))
504 return NDBT_FAILED;
505
506 restarter.waitNodesStartPhase(&nodeId, 1, 3);
507
508 if (restarter.waitNodesNoStart(&nodeId, 1))
509 return NDBT_FAILED;
510
511 if (restarter.dumpStateOneNode(nodeId, val2, 1))
512 return NDBT_FAILED;
513
514 if (restarter.startNodes(&nodeId, 1))
515 return NDBT_FAILED;
516
517 if (restarter.waitNodesStarted(&nodeId, 1))
518 return NDBT_FAILED;
519
520 ctx->stopTest();
521 return NDBT_OK;
522 }
523
runBug15632(NDBT_Context * ctx,NDBT_Step * step)524 int runBug15632(NDBT_Context* ctx, NDBT_Step* step){
525 int result = NDBT_OK;
526 int loops = ctx->getNumLoops();
527 int records = ctx->getNumRecords();
528 NdbRestarter restarter;
529
530 int nodeId = restarter.getDbNodeId(1);
531
532 ndbout << "Restart node " << nodeId << endl;
533
534 if (restarter.restartOneDbNode(nodeId,
535 /** initial */ false,
536 /** nostart */ true,
537 /** abort */ true))
538 return NDBT_FAILED;
539
540 if (restarter.waitNodesNoStart(&nodeId, 1))
541 return NDBT_FAILED;
542
543 if (restarter.insertErrorInNode(nodeId, 7165))
544 return NDBT_FAILED;
545
546 if (restarter.startNodes(&nodeId, 1))
547 return NDBT_FAILED;
548
549 if (restarter.waitNodesStarted(&nodeId, 1))
550 return NDBT_FAILED;
551
552 if (restarter.restartOneDbNode(nodeId,
553 /** initial */ false,
554 /** nostart */ true,
555 /** abort */ true))
556 return NDBT_FAILED;
557
558 if (restarter.waitNodesNoStart(&nodeId, 1))
559 return NDBT_FAILED;
560
561 if (restarter.insertErrorInNode(nodeId, 7171))
562 return NDBT_FAILED;
563
564 if (restarter.startNodes(&nodeId, 1))
565 return NDBT_FAILED;
566
567 if (restarter.waitNodesStarted(&nodeId, 1))
568 return NDBT_FAILED;
569
570 ctx->stopTest();
571 return NDBT_OK;
572 }
573
runBug15685(NDBT_Context * ctx,NDBT_Step * step)574 int runBug15685(NDBT_Context* ctx, NDBT_Step* step){
575
576 Ndb* pNdb = GETNDB(step);
577 HugoOperations hugoOps(*ctx->getTab());
578 NdbRestarter restarter;
579
580 HugoTransactions hugoTrans(*ctx->getTab());
581 if (hugoTrans.loadTable(GETNDB(step), 10) != 0){
582 return NDBT_FAILED;
583 }
584
585 if(hugoOps.startTransaction(pNdb) != 0)
586 goto err;
587
588 if(hugoOps.pkUpdateRecord(pNdb, 0, 1, rand()) != 0)
589 goto err;
590
591 if(hugoOps.execute_NoCommit(pNdb) != 0)
592 goto err;
593
594 if (restarter.insertErrorInAllNodes(5100))
595 return NDBT_FAILED;
596
597 hugoOps.execute_Rollback(pNdb);
598
599 if (restarter.waitClusterStarted() != 0)
600 goto err;
601
602 if (restarter.insertErrorInAllNodes(0))
603 return NDBT_FAILED;
604
605 ctx->stopTest();
606 return NDBT_OK;
607
608 err:
609 ctx->stopTest();
610 return NDBT_FAILED;
611 }
612
613 int
runBug16772(NDBT_Context * ctx,NDBT_Step * step)614 runBug16772(NDBT_Context* ctx, NDBT_Step* step){
615
616 NdbRestarter restarter;
617 if (restarter.getNumDbNodes() < 2)
618 {
619 ctx->stopTest();
620 return NDBT_OK;
621 }
622
623 int aliveNodeId = restarter.getRandomNotMasterNodeId(rand());
624 int deadNodeId = aliveNodeId;
625 while (deadNodeId == aliveNodeId)
626 deadNodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
627
628 if (restarter.insertErrorInNode(aliveNodeId, 930))
629 return NDBT_FAILED;
630
631 if (restarter.restartOneDbNode(deadNodeId,
632 /** initial */ false,
633 /** nostart */ true,
634 /** abort */ true))
635 return NDBT_FAILED;
636
637 if (restarter.waitNodesNoStart(&deadNodeId, 1))
638 return NDBT_FAILED;
639
640 if (restarter.startNodes(&deadNodeId, 1))
641 return NDBT_FAILED;
642
643 // It should now be hanging since we throw away NDB_FAILCONF
644 int ret = restarter.waitNodesStartPhase(&deadNodeId, 1, 3, 10);
645 // So this should fail...i.e it should not reach startphase 3
646
647 // Now send a NDB_FAILCONF for deadNo
648 int dump[] = { 7020, 323, 252, 0 };
649 dump[3] = deadNodeId;
650 if (restarter.dumpStateOneNode(aliveNodeId, dump, 4))
651 return NDBT_FAILED;
652
653 if (restarter.waitNodesStarted(&deadNodeId, 1))
654 return NDBT_FAILED;
655
656 return ret ? NDBT_OK : NDBT_FAILED;
657 }
658
659 int
runBug18414(NDBT_Context * ctx,NDBT_Step * step)660 runBug18414(NDBT_Context* ctx, NDBT_Step* step){
661
662 NdbRestarter restarter;
663 if (restarter.getNumDbNodes() < 2)
664 {
665 ctx->stopTest();
666 return NDBT_OK;
667 }
668
669 Ndb* pNdb = GETNDB(step);
670 HugoOperations hugoOps(*ctx->getTab());
671 HugoTransactions hugoTrans(*ctx->getTab());
672 int loop = 0;
673 do
674 {
675 if(hugoOps.startTransaction(pNdb) != 0)
676 goto err;
677
678 if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0)
679 goto err;
680
681 if(hugoOps.execute_NoCommit(pNdb) != 0)
682 goto err;
683
684 int node1 = hugoOps.getTransaction()->getConnectedNodeId();
685 int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
686
687 if (node1 == -1 || node2 == -1)
688 break;
689
690 if (loop & 1)
691 {
692 if (restarter.insertErrorInNode(node1, 8050))
693 goto err;
694 }
695
696 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
697 if (restarter.dumpStateOneNode(node2, val2, 2))
698 goto err;
699
700 if (restarter.insertErrorInNode(node2, 5003))
701 goto err;
702
703 int res= hugoOps.execute_Rollback(pNdb);
704
705 if (restarter.waitNodesNoStart(&node2, 1) != 0)
706 goto err;
707
708 if (restarter.insertErrorInAllNodes(0))
709 goto err;
710
711 if (restarter.startNodes(&node2, 1) != 0)
712 goto err;
713
714 if (restarter.waitClusterStarted() != 0)
715 goto err;
716
717 if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0)
718 goto err;
719
720 hugoOps.closeTransaction(pNdb);
721
722 } while(++loop < 5);
723
724 return NDBT_OK;
725
726 err:
727 hugoOps.closeTransaction(pNdb);
728 return NDBT_FAILED;
729 }
730
731 int
runBug18612(NDBT_Context * ctx,NDBT_Step * step)732 runBug18612(NDBT_Context* ctx, NDBT_Step* step){
733
734 // Assume two replicas
735 NdbRestarter restarter;
736 if (restarter.getNumDbNodes() < 2)
737 {
738 ctx->stopTest();
739 return NDBT_OK;
740 }
741
742 Uint32 cnt = restarter.getNumDbNodes();
743
744 for(int loop = 0; loop < ctx->getNumLoops(); loop++)
745 {
746 int partition0[256];
747 int partition1[256];
748 bzero(partition0, sizeof(partition0));
749 bzero(partition1, sizeof(partition1));
750 Bitmask<4> nodesmask;
751
752 Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
753 for (Uint32 i = 0; i<cnt/2; i++)
754 {
755 do {
756 int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
757 if (tmp == -1)
758 {
759 ctx->stopTest();
760 return NDBT_OK;
761 }
762 node1 = tmp;
763 } while(nodesmask.get(node1));
764
765 partition0[i] = node1;
766 partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
767
768 ndbout_c("nodes %d %d", node1, partition1[i]);
769
770 assert(!nodesmask.get(node1));
771 assert(!nodesmask.get(partition1[i]));
772 nodesmask.set(node1);
773 nodesmask.set(partition1[i]);
774 }
775
776 ndbout_c("done");
777
778 int dump[255];
779 dump[0] = DumpStateOrd::NdbcntrStopNodes;
780 memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
781
782 Uint32 master = restarter.getMasterNodeId();
783
784 if (restarter.dumpStateOneNode(master, dump, 1+cnt/2))
785 return NDBT_FAILED;
786
787 if (restarter.waitNodesNoStart(partition0, cnt/2))
788 return NDBT_FAILED;
789
790 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
791
792 if (restarter.dumpStateAllNodes(val2, 2))
793 return NDBT_FAILED;
794
795 if (restarter.insertErrorInAllNodes(932))
796 return NDBT_FAILED;
797
798 dump[0] = 9000;
799 memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
800 for (Uint32 i = 0; i<cnt/2; i++)
801 if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
802 return NDBT_FAILED;
803
804 dump[0] = 9000;
805 memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
806 for (Uint32 i = 0; i<cnt/2; i++)
807 if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
808 return NDBT_FAILED;
809
810 if (restarter.startNodes(partition0, cnt/2))
811 return NDBT_FAILED;
812
813 if (restarter.waitNodesStartPhase(partition0, cnt/2, 2))
814 return NDBT_FAILED;
815
816 dump[0] = 9001;
817 for (Uint32 i = 0; i<cnt/2; i++)
818 if (restarter.dumpStateAllNodes(dump, 2))
819 return NDBT_FAILED;
820
821 if (restarter.waitNodesNoStart(partition0, cnt/2))
822 return NDBT_FAILED;
823
824 for (Uint32 i = 0; i<cnt/2; i++)
825 if (restarter.restartOneDbNode(partition0[i], true, true, true))
826 return NDBT_FAILED;
827
828 if (restarter.waitNodesNoStart(partition0, cnt/2))
829 return NDBT_FAILED;
830
831 if (restarter.startAll())
832 return NDBT_FAILED;
833
834 if (restarter.waitClusterStarted())
835 return NDBT_FAILED;
836 }
837 return NDBT_OK;
838 }
839
840 int
runBug18612SR(NDBT_Context * ctx,NDBT_Step * step)841 runBug18612SR(NDBT_Context* ctx, NDBT_Step* step){
842
843 // Assume two replicas
844 NdbRestarter restarter;
845 if (restarter.getNumDbNodes() < 2)
846 {
847 ctx->stopTest();
848 return NDBT_OK;
849 }
850
851 Uint32 cnt = restarter.getNumDbNodes();
852
853 for(int loop = 0; loop < ctx->getNumLoops(); loop++)
854 {
855 int partition0[256];
856 int partition1[256];
857 bzero(partition0, sizeof(partition0));
858 bzero(partition1, sizeof(partition1));
859 Bitmask<4> nodesmask;
860
861 Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
862 for (Uint32 i = 0; i<cnt/2; i++)
863 {
864 do {
865 int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
866 if (tmp == -1)
867 break;
868 node1 = tmp;
869 } while(nodesmask.get(node1));
870
871 partition0[i] = node1;
872 partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
873
874 ndbout_c("nodes %d %d", node1, partition1[i]);
875
876 assert(!nodesmask.get(node1));
877 assert(!nodesmask.get(partition1[i]));
878 nodesmask.set(node1);
879 nodesmask.set(partition1[i]);
880 }
881
882 ndbout_c("done");
883
884 if (restarter.restartAll(false, true, false))
885 return NDBT_FAILED;
886
887 int dump[255];
888 dump[0] = 9000;
889 memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
890 for (Uint32 i = 0; i<cnt/2; i++)
891 if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
892 return NDBT_FAILED;
893
894 dump[0] = 9000;
895 memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
896 for (Uint32 i = 0; i<cnt/2; i++)
897 if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
898 return NDBT_FAILED;
899
900 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
901
902 if (restarter.dumpStateAllNodes(val2, 2))
903 return NDBT_FAILED;
904
905 if (restarter.insertErrorInAllNodes(932))
906 return NDBT_FAILED;
907
908 if (restarter.startAll())
909 return NDBT_FAILED;
910
911 if (restarter.waitClusterStartPhase(2))
912 return NDBT_FAILED;
913
914 dump[0] = 9001;
915 for (Uint32 i = 0; i<cnt/2; i++)
916 if (restarter.dumpStateAllNodes(dump, 2))
917 return NDBT_FAILED;
918
919 if (restarter.waitClusterNoStart(30))
920 if (restarter.waitNodesNoStart(partition0, cnt/2, 10))
921 if (restarter.waitNodesNoStart(partition1, cnt/2, 10))
922 return NDBT_FAILED;
923
924 if (restarter.startAll())
925 return NDBT_FAILED;
926
927 if (restarter.waitClusterStarted())
928 return NDBT_FAILED;
929 }
930 return NDBT_OK;
931 }
932
runBug20185(NDBT_Context * ctx,NDBT_Step * step)933 int runBug20185(NDBT_Context* ctx, NDBT_Step* step){
934 int result = NDBT_OK;
935 int loops = ctx->getNumLoops();
936 int records = ctx->getNumRecords();
937 NdbRestarter restarter;
938 HugoOperations hugoOps(*ctx->getTab());
939 Ndb* pNdb = GETNDB(step);
940
941 const int masterNode = restarter.getMasterNodeId();
942
943 int dump[] = { 7090, 20 } ;
944 if (restarter.dumpStateAllNodes(dump, 2))
945 return NDBT_FAILED;
946
947 NdbSleep_MilliSleep(3000);
948 Vector<int> nodes;
949 for (Uint32 i = 0; i<restarter.getNumDbNodes(); i++)
950 nodes.push_back(restarter.getDbNodeId(i));
951
952 retry:
953 if(hugoOps.startTransaction(pNdb) != 0)
954 return NDBT_FAILED;
955
956 if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0)
957 return NDBT_FAILED;
958
959 if (hugoOps.execute_NoCommit(pNdb) != 0)
960 return NDBT_FAILED;
961
962 const int node = hugoOps.getTransaction()->getConnectedNodeId();
963 if (node != masterNode)
964 {
965 hugoOps.closeTransaction(pNdb);
966 goto retry;
967 }
968
969 int nodeId;
970 do {
971 nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
972 } while (nodeId == node);
973
974 ndbout_c("7031 to %d", nodeId);
975 if (restarter.insertErrorInNode(nodeId, 7031))
976 return NDBT_FAILED;
977
978 for (Uint32 i = 0; i<nodes.size(); i++)
979 {
980 if (nodes[i] != nodeId)
981 if (restarter.insertErrorInNode(nodes[i], 7030))
982 return NDBT_FAILED;
983 }
984
985 NdbSleep_MilliSleep(500);
986
987 if (hugoOps.execute_Commit(pNdb) == 0)
988 return NDBT_FAILED;
989
990 NdbSleep_MilliSleep(3000);
991
992 restarter.waitClusterStarted();
993
994 if (restarter.dumpStateAllNodes(dump, 1))
995 return NDBT_FAILED;
996
997 return NDBT_OK;
998 }
999
runBug24717(NDBT_Context * ctx,NDBT_Step * step)1000 int runBug24717(NDBT_Context* ctx, NDBT_Step* step){
1001 int result = NDBT_OK;
1002 int loops = ctx->getNumLoops();
1003 int records = ctx->getNumRecords();
1004 NdbRestarter restarter;
1005 Ndb* pNdb = GETNDB(step);
1006
1007 HugoTransactions hugoTrans(*ctx->getTab());
1008
1009 int dump[] = { 9002, 0 } ;
1010 Uint32 ownNode = refToNode(pNdb->getReference());
1011 dump[1] = ownNode;
1012
1013 for (; loops; loops --)
1014 {
1015 int nodeId = restarter.getRandomNotMasterNodeId(rand());
1016 restarter.restartOneDbNode(nodeId, false, true, true);
1017 restarter.waitNodesNoStart(&nodeId, 1);
1018
1019 if (restarter.dumpStateOneNode(nodeId, dump, 2))
1020 return NDBT_FAILED;
1021
1022 restarter.startNodes(&nodeId, 1);
1023
1024 do {
1025 for (Uint32 i = 0; i < 100; i++)
1026 {
1027 hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
1028 }
1029 } while (restarter.waitClusterStarted(5) != 0);
1030 }
1031
1032 return NDBT_OK;
1033 }
1034
1035 int
runBug29364(NDBT_Context * ctx,NDBT_Step * step)1036 runBug29364(NDBT_Context* ctx, NDBT_Step* step){
1037 int result = NDBT_OK;
1038 int loops = ctx->getNumLoops();
1039 int records = ctx->getNumRecords();
1040 NdbRestarter restarter;
1041 Ndb* pNdb = GETNDB(step);
1042
1043 HugoTransactions hugoTrans(*ctx->getTab());
1044
1045 if (restarter.getNumDbNodes() < 4)
1046 return NDBT_OK;
1047
1048 int dump0[] = { 9000, 0 } ;
1049 int dump1[] = { 9001, 0 } ;
1050 Uint32 ownNode = refToNode(pNdb->getReference());
1051 dump0[1] = ownNode;
1052
1053 for (; loops; loops --)
1054 {
1055 int node0 = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1056 int node1 = restarter.getRandomNodeOtherNodeGroup(node0, rand());
1057
1058 restarter.restartOneDbNode(node0, false, true, true);
1059 restarter.waitNodesNoStart(&node0, 1);
1060 restarter.startNodes(&node0, 1);
1061 restarter.waitClusterStarted();
1062
1063 restarter.restartOneDbNode(node1, false, true, true);
1064 restarter.waitNodesNoStart(&node1, 1);
1065 if (restarter.dumpStateOneNode(node1, dump0, 2))
1066 return NDBT_FAILED;
1067
1068 restarter.startNodes(&node1, 1);
1069
1070 do {
1071
1072 for (Uint32 i = 0; i < 100; i++)
1073 {
1074 hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
1075 }
1076 } while (restarter.waitClusterStarted(5) != 0);
1077
1078 if (restarter.dumpStateOneNode(node1, dump1, 1))
1079 return NDBT_FAILED;
1080 }
1081
1082 return NDBT_OK;
1083 }
1084
runBug25364(NDBT_Context * ctx,NDBT_Step * step)1085 int runBug25364(NDBT_Context* ctx, NDBT_Step* step){
1086 int result = NDBT_OK;
1087 NdbRestarter restarter;
1088 Ndb* pNdb = GETNDB(step);
1089 int loops = ctx->getNumLoops();
1090
1091 if (restarter.getNumDbNodes() < 4)
1092 return NDBT_OK;
1093
1094 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1095
1096 for (; loops; loops --)
1097 {
1098 int master = restarter.getMasterNodeId();
1099 int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
1100 int second = restarter.getRandomNodeSameNodeGroup(victim, rand());
1101
1102 int dump[] = { 935, victim } ;
1103 if (restarter.dumpStateOneNode(master, dump, 2))
1104 return NDBT_FAILED;
1105
1106 if (restarter.dumpStateOneNode(master, val2, 2))
1107 return NDBT_FAILED;
1108
1109 if (restarter.restartOneDbNode(second, false, true, true))
1110 return NDBT_FAILED;
1111
1112 int nodes[2] = { master, second };
1113 if (restarter.waitNodesNoStart(nodes, 2))
1114 return NDBT_FAILED;
1115
1116 restarter.startNodes(nodes, 2);
1117
1118 if (restarter.waitNodesStarted(nodes, 2))
1119 return NDBT_FAILED;
1120 }
1121
1122 return NDBT_OK;
1123 }
1124
1125 int
runBug21271(NDBT_Context * ctx,NDBT_Step * step)1126 runBug21271(NDBT_Context* ctx, NDBT_Step* step){
1127 int result = NDBT_OK;
1128 int loops = ctx->getNumLoops();
1129 int records = ctx->getNumRecords();
1130 NdbRestarter restarter;
1131 HugoOperations hugoOps(*ctx->getTab());
1132 Ndb* pNdb = GETNDB(step);
1133
1134 const int masterNode = restarter.getMasterNodeId();
1135 const int nodeId = restarter.getRandomNodeSameNodeGroup(masterNode, rand());
1136
1137 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1138 if (restarter.dumpStateOneNode(nodeId, val2, 2))
1139 return NDBT_FAILED;
1140
1141 Uint32 tableId = ctx->getTab()->getTableId();
1142 int dump[] = { DumpStateOrd::LqhErrorInsert5042, 0, 5044 };
1143 dump[1] = tableId;
1144
1145 if (restarter.dumpStateOneNode(nodeId, dump, 3))
1146 return NDBT_FAILED;
1147
1148 restarter.waitNodesNoStart(&nodeId, 1);
1149 ctx->stopTest();
1150
1151 restarter.startNodes(&nodeId, 1);
1152
1153 if (restarter.waitClusterStarted() != 0)
1154 return NDBT_FAILED;
1155
1156 return NDBT_OK;
1157 return NDBT_OK;
1158 }
1159
1160 int
runBug24543(NDBT_Context * ctx,NDBT_Step * step)1161 runBug24543(NDBT_Context* ctx, NDBT_Step* step){
1162 NdbRestarter restarter;
1163
1164 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1165 if (restarter.dumpStateAllNodes(val2, 2))
1166 return NDBT_FAILED;
1167
1168 int nodes[2];
1169 nodes[0] = restarter.getMasterNodeId();
1170 restarter.insertErrorInNode(nodes[0], 934);
1171
1172 nodes[1] = restarter.getRandomNodeOtherNodeGroup(nodes[0], rand());
1173 if (nodes[1] == -1)
1174 {
1175 nodes[1] = restarter.getRandomNodeSameNodeGroup(nodes[0], rand());
1176 }
1177
1178 restarter.restartOneDbNode(nodes[1], false, true, true);
1179 if (restarter.waitNodesNoStart(nodes, 2))
1180 return NDBT_FAILED;
1181
1182 restarter.startNodes(nodes, 2);
1183 if (restarter.waitNodesStarted(nodes, 2))
1184 {
1185 return NDBT_FAILED;
1186 }
1187 return NDBT_OK;
1188 }
1189
runBug25468(NDBT_Context * ctx,NDBT_Step * step)1190 int runBug25468(NDBT_Context* ctx, NDBT_Step* step){
1191
1192 int result = NDBT_OK;
1193 int loops = ctx->getNumLoops();
1194 int records = ctx->getNumRecords();
1195 NdbRestarter restarter;
1196
1197 for (int i = 0; i<loops; i++)
1198 {
1199 int master = restarter.getMasterNodeId();
1200 int node1, node2;
1201 switch(i % 5){
1202 case 0:
1203 node1 = master;
1204 node2 = restarter.getRandomNodeSameNodeGroup(master, rand());
1205 break;
1206 case 1:
1207 node1 = restarter.getRandomNodeSameNodeGroup(master, rand());
1208 node2 = master;
1209 break;
1210 case 2:
1211 case 3:
1212 case 4:
1213 node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1214 if (node1 == -1)
1215 node1 = master;
1216 node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
1217 break;
1218 }
1219
1220 ndbout_c("node1: %d node2: %d master: %d", node1, node2, master);
1221
1222 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1223
1224 if (restarter.dumpStateOneNode(node2, val2, 2))
1225 return NDBT_FAILED;
1226
1227 if (restarter.insertErrorInNode(node1, 7178))
1228 return NDBT_FAILED;
1229
1230 int val1 = 7099;
1231 if (restarter.dumpStateOneNode(master, &val1, 1))
1232 return NDBT_FAILED;
1233
1234 if (restarter.waitNodesNoStart(&node2, 1))
1235 return NDBT_FAILED;
1236
1237 if (restarter.startAll())
1238 return NDBT_FAILED;
1239
1240 if (restarter.waitClusterStarted())
1241 return NDBT_FAILED;
1242 }
1243
1244 return NDBT_OK;
1245 }
1246
runBug25554(NDBT_Context * ctx,NDBT_Step * step)1247 int runBug25554(NDBT_Context* ctx, NDBT_Step* step){
1248
1249 int result = NDBT_OK;
1250 int loops = ctx->getNumLoops();
1251 int records = ctx->getNumRecords();
1252 NdbRestarter restarter;
1253
1254 if (restarter.getNumDbNodes() < 4)
1255 return NDBT_OK;
1256
1257 for (int i = 0; i<loops; i++)
1258 {
1259 int master = restarter.getMasterNodeId();
1260 int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1261 restarter.restartOneDbNode(node1, false, true, true);
1262
1263 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1264
1265 if (restarter.dumpStateOneNode(master, val2, 2))
1266 return NDBT_FAILED;
1267
1268 if (restarter.insertErrorInNode(master, 7141))
1269 return NDBT_FAILED;
1270
1271 if (restarter.waitNodesNoStart(&node1, 1))
1272 return NDBT_FAILED;
1273
1274 if (restarter.dumpStateOneNode(node1, val2, 2))
1275 return NDBT_FAILED;
1276
1277 if (restarter.insertErrorInNode(node1, 932))
1278 return NDBT_FAILED;
1279
1280 if (restarter.startNodes(&node1, 1))
1281 return NDBT_FAILED;
1282
1283 int nodes[] = { master, node1 };
1284 if (restarter.waitNodesNoStart(nodes, 2))
1285 return NDBT_FAILED;
1286
1287 if (restarter.startNodes(nodes, 2))
1288 return NDBT_FAILED;
1289
1290 if (restarter.waitClusterStarted())
1291 return NDBT_FAILED;
1292 }
1293
1294 return NDBT_OK;
1295 }
1296
runBug25984(NDBT_Context * ctx,NDBT_Step * step)1297 int runBug25984(NDBT_Context* ctx, NDBT_Step* step){
1298
1299 int result = NDBT_OK;
1300 int loops = ctx->getNumLoops();
1301 int records = ctx->getNumRecords();
1302 NdbRestarter restarter;
1303
1304 if (restarter.getNumDbNodes() < 2)
1305 return NDBT_OK;
1306
1307 if (restarter.restartAll(true, true, true))
1308 return NDBT_FAILED;
1309
1310 if (restarter.waitClusterNoStart())
1311 return NDBT_FAILED;
1312
1313 if (restarter.startAll())
1314 return NDBT_FAILED;
1315
1316 if (restarter.waitClusterStarted())
1317 return NDBT_FAILED;
1318
1319 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1320 int master = restarter.getMasterNodeId();
1321 int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
1322 if (victim == -1)
1323 victim = restarter.getRandomNodeSameNodeGroup(master, rand());
1324
1325 restarter.restartOneDbNode(victim, false, true, true);
1326
1327 for (Uint32 i = 0; i<6; i++)
1328 {
1329 ndbout_c("Loop: %d", i);
1330 if (restarter.waitNodesNoStart(&victim, 1))
1331 return NDBT_FAILED;
1332
1333 if (restarter.dumpStateOneNode(victim, val2, 2))
1334 return NDBT_FAILED;
1335
1336 if (restarter.insertErrorInNode(victim, 7016))
1337 return NDBT_FAILED;
1338
1339 if (restarter.startNodes(&victim, 1))
1340 return NDBT_FAILED;
1341
1342 NdbSleep_SecSleep(3);
1343 }
1344
1345 if (restarter.waitNodesNoStart(&victim, 1))
1346 return NDBT_FAILED;
1347
1348 if (restarter.dumpStateOneNode(victim, val2, 2))
1349 return NDBT_FAILED;
1350
1351 if (restarter.insertErrorInNode(victim, 7170))
1352 return NDBT_FAILED;
1353
1354 if (restarter.startNodes(&victim, 1))
1355 return NDBT_FAILED;
1356
1357 if (restarter.waitNodesNoStart(&victim, 1))
1358 return NDBT_FAILED;
1359
1360 if (restarter.restartAll(false, true, true))
1361 return NDBT_FAILED;
1362
1363 if (restarter.insertErrorInAllNodes(932))
1364 return NDBT_FAILED;
1365
1366 if (restarter.insertErrorInNode(master, 7170))
1367 return NDBT_FAILED;
1368
1369 if (restarter.dumpStateAllNodes(val2, 2))
1370 return NDBT_FAILED;
1371
1372 restarter.startNodes(&master, 1);
1373 NdbSleep_MilliSleep(3000);
1374 restarter.startAll();
1375
1376 if (restarter.waitClusterNoStart())
1377 return NDBT_FAILED;
1378
1379 if (restarter.restartOneDbNode(victim, true, true, true))
1380 return NDBT_FAILED;
1381
1382 if (restarter.startAll())
1383 return NDBT_FAILED;
1384
1385 if (restarter.waitClusterStarted())
1386 return NDBT_FAILED;
1387
1388 return NDBT_OK;
1389 }
1390
1391 int
runBug26457(NDBT_Context * ctx,NDBT_Step * step)1392 runBug26457(NDBT_Context* ctx, NDBT_Step* step)
1393 {
1394 NdbRestarter res;
1395 if (res.getNumDbNodes() < 4)
1396 return NDBT_OK;
1397
1398 int loops = ctx->getNumLoops();
1399 while (loops --)
1400 {
1401 retry:
1402 int master = res.getMasterNodeId();
1403 int next = res.getNextMasterNodeId(master);
1404
1405 ndbout_c("master: %d next: %d", master, next);
1406
1407 if (res.getNodeGroup(master) == res.getNodeGroup(next))
1408 {
1409 res.restartOneDbNode(next, false, false, true);
1410 if (res.waitClusterStarted())
1411 return NDBT_FAILED;
1412 goto retry;
1413 }
1414
1415 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 2 };
1416
1417 if (res.dumpStateOneNode(next, val2, 2))
1418 return NDBT_FAILED;
1419
1420 if (res.insertErrorInNode(next, 7180))
1421 return NDBT_FAILED;
1422
1423 res.restartOneDbNode(master, false, false, true);
1424 if (res.waitClusterStarted())
1425 return NDBT_FAILED;
1426 }
1427
1428 return NDBT_OK;
1429 }
1430
1431 int
runBug26481(NDBT_Context * ctx,NDBT_Step * step)1432 runBug26481(NDBT_Context* ctx, NDBT_Step* step)
1433 {
1434
1435 int result = NDBT_OK;
1436 int loops = ctx->getNumLoops();
1437 int records = ctx->getNumRecords();
1438 NdbRestarter res;
1439
1440 int node = res.getRandomNotMasterNodeId(rand());
1441 ndbout_c("node: %d", node);
1442 if (res.restartOneDbNode(node, true, true, true))
1443 return NDBT_FAILED;
1444
1445 if (res.waitNodesNoStart(&node, 1))
1446 return NDBT_FAILED;
1447
1448 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1449 if (res.dumpStateOneNode(node, val2, 2))
1450 return NDBT_FAILED;
1451
1452 if (res.insertErrorInNode(node, 7018))
1453 return NDBT_FAILED;
1454
1455 if (res.startNodes(&node, 1))
1456 return NDBT_FAILED;
1457
1458 res.waitNodesStartPhase(&node, 1, 3);
1459
1460 if (res.waitNodesNoStart(&node, 1))
1461 return NDBT_FAILED;
1462
1463 res.startNodes(&node, 1);
1464
1465 if (res.waitClusterStarted())
1466 return NDBT_FAILED;
1467
1468 return NDBT_OK;
1469 }
1470
1471 int
runBug26450(NDBT_Context * ctx,NDBT_Step * step)1472 runBug26450(NDBT_Context* ctx, NDBT_Step* step)
1473 {
1474 Uint32 i;
1475 int result = NDBT_OK;
1476 int loops = ctx->getNumLoops();
1477 int records = ctx->getNumRecords();
1478 NdbRestarter res;
1479 Ndb* pNdb = GETNDB(step);
1480
1481 int node = res.getRandomNotMasterNodeId(rand());
1482 Vector<int> nodes;
1483 for (unsigned i = 0; i<res.getNumDbNodes(); i++)
1484 {
1485 if (res.getDbNodeId(i) != node)
1486 nodes.push_back(res.getDbNodeId(i));
1487 }
1488
1489 if (res.restartAll())
1490 return NDBT_FAILED;
1491
1492 if (res.waitClusterStarted())
1493 return NDBT_FAILED;
1494
1495 ndbout_c("node: %d", node);
1496 if (res.restartOneDbNode(node, false, true, true))
1497 return NDBT_FAILED;
1498
1499 if (res.waitNodesNoStart(&node, 1))
1500 return NDBT_FAILED;
1501
1502 if (runClearTable(ctx, step))
1503 return NDBT_FAILED;
1504
1505 for (i = 0; i < 2; i++)
1506 {
1507 if (res.restartAll(false, true, i > 0))
1508 return NDBT_FAILED;
1509
1510 if (res.waitClusterNoStart())
1511 return NDBT_FAILED;
1512
1513 if (res.startNodes(nodes.getBase(), nodes.size()))
1514 return NDBT_FAILED;
1515
1516 if (res.waitNodesStarted(nodes.getBase(), nodes.size()))
1517 return NDBT_FAILED;
1518 }
1519
1520 if (res.startNodes(&node, 1))
1521 return NDBT_FAILED;
1522
1523 if (res.waitNodesStarted(&node, 1))
1524 return NDBT_FAILED;
1525
1526 HugoTransactions trans (* ctx->getTab());
1527 if (trans.selectCount(pNdb) != 0)
1528 return NDBT_FAILED;
1529
1530 return NDBT_OK;
1531 }
1532
1533 int
runBug27003(NDBT_Context * ctx,NDBT_Step * step)1534 runBug27003(NDBT_Context* ctx, NDBT_Step* step)
1535 {
1536 int result = NDBT_OK;
1537 int loops = ctx->getNumLoops();
1538 int records = ctx->getNumRecords();
1539 NdbRestarter res;
1540
1541 static const int errnos[] = { 4025, 4026, 4027, 4028, 0 };
1542
1543 int node = res.getRandomNotMasterNodeId(rand());
1544 ndbout_c("node: %d", node);
1545 if (res.restartOneDbNode(node, true, true, true))
1546 return NDBT_FAILED;
1547
1548 Uint32 pos = 0;
1549 for (Uint32 i = 0; i<loops; i++)
1550 {
1551 while (errnos[pos] != 0)
1552 {
1553 ndbout_c("Tesing err: %d", errnos[pos]);
1554
1555 if (res.waitNodesNoStart(&node, 1))
1556 return NDBT_FAILED;
1557
1558 if (res.insertErrorInNode(node, 1000))
1559 return NDBT_FAILED;
1560
1561 if (res.insertErrorInNode(node, errnos[pos]))
1562 return NDBT_FAILED;
1563
1564 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 3 };
1565 if (res.dumpStateOneNode(node, val2, 2))
1566 return NDBT_FAILED;
1567
1568 res.startNodes(&node, 1);
1569 NdbSleep_SecSleep(3);
1570 pos++;
1571 }
1572 pos = 0;
1573 }
1574
1575 if (res.waitNodesNoStart(&node, 1))
1576 return NDBT_FAILED;
1577
1578 res.startNodes(&node, 1);
1579 if (res.waitClusterStarted())
1580 return NDBT_FAILED;
1581
1582 return NDBT_OK;
1583 }
1584
1585
1586 int
runBug27283(NDBT_Context * ctx,NDBT_Step * step)1587 runBug27283(NDBT_Context* ctx, NDBT_Step* step)
1588 {
1589 int result = NDBT_OK;
1590 int loops = ctx->getNumLoops();
1591 int records = ctx->getNumRecords();
1592 NdbRestarter res;
1593
1594 if (res.getNumDbNodes() < 2)
1595 {
1596 return NDBT_OK;
1597 }
1598
1599 static const int errnos[] = { 7181, 7182, 0 };
1600
1601 Uint32 pos = 0;
1602 for (Uint32 i = 0; i<loops; i++)
1603 {
1604 while (errnos[pos] != 0)
1605 {
1606 int master = res.getMasterNodeId();
1607 int next = res.getNextMasterNodeId(master);
1608 int next2 = res.getNextMasterNodeId(next);
1609
1610 int node = (i & 1) ? next : next2;
1611 ndbout_c("Tesing err: %d", errnos[pos]);
1612 if (res.insertErrorInNode(next, errnos[pos]))
1613 return NDBT_FAILED;
1614
1615 NdbSleep_SecSleep(3);
1616
1617 if (res.waitClusterStarted())
1618 return NDBT_FAILED;
1619
1620 pos++;
1621 }
1622 pos = 0;
1623 }
1624
1625 return NDBT_OK;
1626 }
1627
1628 int
runBug27466(NDBT_Context * ctx,NDBT_Step * step)1629 runBug27466(NDBT_Context* ctx, NDBT_Step* step)
1630 {
1631 int result = NDBT_OK;
1632 int loops = ctx->getNumLoops();
1633 int records = ctx->getNumRecords();
1634 NdbRestarter res;
1635
1636 if (res.getNumDbNodes() < 2)
1637 {
1638 return NDBT_OK;
1639 }
1640
1641 Uint32 pos = 0;
1642 for (Uint32 i = 0; i<loops; i++)
1643 {
1644 int node1 = res.getDbNodeId(rand() % res.getNumDbNodes());
1645 int node2 = node1;
1646 while (node1 == node2)
1647 {
1648 node2 = res.getDbNodeId(rand() % res.getNumDbNodes());
1649 }
1650
1651 ndbout_c("nodes %u %u", node1, node2);
1652
1653 if (res.restartOneDbNode(node1, false, true, true))
1654 return NDBT_FAILED;
1655
1656 if (res.waitNodesNoStart(&node1, 1))
1657 return NDBT_FAILED;
1658
1659 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1660 if (res.dumpStateOneNode(node1, val2, 2))
1661 return NDBT_FAILED;
1662
1663 if (res.insertErrorInNode(node2, 8039))
1664 return NDBT_FAILED;
1665
1666 res.startNodes(&node1, 1);
1667 NdbSleep_SecSleep(3);
1668 if (res.waitNodesNoStart(&node1, 1))
1669 return NDBT_FAILED;
1670 NdbSleep_SecSleep(5); // Wait for delayed INCL_NODECONF to arrive
1671
1672 res.startNodes(&node1, 1);
1673 if (res.waitClusterStarted())
1674 return NDBT_FAILED;
1675 }
1676
1677 return NDBT_OK;
1678 }
1679
1680 int
runBug28023(NDBT_Context * ctx,NDBT_Step * step)1681 runBug28023(NDBT_Context* ctx, NDBT_Step* step)
1682 {
1683 int result = NDBT_OK;
1684 int loops = ctx->getNumLoops();
1685 int records = ctx->getNumRecords();
1686 Ndb* pNdb = GETNDB(step);
1687 NdbRestarter res;
1688
1689 if (res.getNumDbNodes() < 2)
1690 {
1691 return NDBT_OK;
1692 }
1693
1694
1695 HugoTransactions hugoTrans(*ctx->getTab());
1696 if (hugoTrans.loadTable(pNdb, records) != 0){
1697 return NDBT_FAILED;
1698 }
1699
1700 if (hugoTrans.clearTable(pNdb, records) != 0)
1701 {
1702 return NDBT_FAILED;
1703 }
1704
1705 for (Uint32 i = 0; i<loops; i++)
1706 {
1707 int node1 = res.getDbNodeId(rand() % res.getNumDbNodes());
1708
1709 if (res.restartOneDbNode2(node1,
1710 NdbRestarter::NRRF_ABORT |
1711 NdbRestarter::NRRF_NOSTART))
1712 return NDBT_FAILED;
1713
1714 if (res.waitNodesNoStart(&node1, 1))
1715 return NDBT_FAILED;
1716
1717 if (hugoTrans.loadTable(pNdb, records) != 0){
1718 return NDBT_FAILED;
1719 }
1720
1721 if (hugoTrans.clearTable(pNdb, records) != 0)
1722 {
1723 return NDBT_FAILED;
1724 }
1725
1726 res.startNodes(&node1, 1);
1727 if (res.waitClusterStarted())
1728 return NDBT_FAILED;
1729
1730 if (hugoTrans.loadTable(pNdb, records) != 0){
1731 return NDBT_FAILED;
1732 }
1733
1734 if (hugoTrans.scanUpdateRecords(pNdb, records) != 0)
1735 return NDBT_FAILED;
1736
1737 if (hugoTrans.clearTable(pNdb, records) != 0)
1738 {
1739 return NDBT_FAILED;
1740 }
1741 }
1742
1743 return NDBT_OK;
1744 }
1745
1746
1747 int
runBug28717(NDBT_Context * ctx,NDBT_Step * step)1748 runBug28717(NDBT_Context* ctx, NDBT_Step* step)
1749 {
1750 int result = NDBT_OK;
1751 int loops = ctx->getNumLoops();
1752 int records = ctx->getNumRecords();
1753 Ndb* pNdb = GETNDB(step);
1754 NdbRestarter res;
1755
1756 if (res.getNumDbNodes() < 4)
1757 {
1758 return NDBT_OK;
1759 }
1760
1761 int master = res.getMasterNodeId();
1762 int node0 = res.getRandomNodeOtherNodeGroup(master, rand());
1763 int node1 = res.getRandomNodeSameNodeGroup(node0, rand());
1764
1765 ndbout_c("master: %d node0: %d node1: %d", master, node0, node1);
1766
1767 if (res.restartOneDbNode(node0, false, true, true))
1768 {
1769 return NDBT_FAILED;
1770 }
1771
1772 {
1773 int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1774 NdbLogEventHandle handle =
1775 ndb_mgm_create_logevent_handle(res.handle, filter);
1776
1777
1778 int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1779 struct ndb_logevent event;
1780
1781 for (Uint32 i = 0; i<3; i++)
1782 {
1783 res.dumpStateOneNode(master, dump, 1);
1784 while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1785 event.type != NDB_LE_LocalCheckpointStarted);
1786 while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1787 event.type != NDB_LE_LocalCheckpointCompleted);
1788 }
1789 }
1790
1791 if (res.waitNodesNoStart(&node0, 1))
1792 return NDBT_FAILED;
1793
1794 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1795
1796 if (res.dumpStateOneNode(node0, val2, 2))
1797 return NDBT_FAILED;
1798
1799 if (res.insertErrorInNode(node0, 5010))
1800 return NDBT_FAILED;
1801
1802 if (res.insertErrorInNode(node1, 1001))
1803 return NDBT_FAILED;
1804
1805 if (res.startNodes(&node0, 1))
1806 return NDBT_FAILED;
1807
1808 NdbSleep_SecSleep(3);
1809
1810 if (res.insertErrorInNode(node1, 0))
1811 return NDBT_FAILED;
1812
1813 if (res.waitNodesNoStart(&node0, 1))
1814 return NDBT_FAILED;
1815
1816 if (res.startNodes(&node0, 1))
1817 return NDBT_FAILED;
1818
1819 if (res.waitClusterStarted())
1820 return NDBT_FAILED;
1821
1822 return NDBT_OK;
1823 }
1824
1825 int
runBug31525(NDBT_Context * ctx,NDBT_Step * step)1826 runBug31525(NDBT_Context* ctx, NDBT_Step* step)
1827 {
1828 int result = NDBT_OK;
1829 int loops = ctx->getNumLoops();
1830 int records = ctx->getNumRecords();
1831 Ndb* pNdb = GETNDB(step);
1832 NdbRestarter res;
1833
1834 if (res.getNumDbNodes() < 2)
1835 {
1836 return NDBT_OK;
1837 }
1838
1839 int nodes[2];
1840 nodes[0] = res.getMasterNodeId();
1841 nodes[1] = res.getNextMasterNodeId(nodes[0]);
1842
1843 while (res.getNodeGroup(nodes[0]) != res.getNodeGroup(nodes[1]))
1844 {
1845 ndbout_c("Restarting %u as it not in same node group as %u",
1846 nodes[1], nodes[0]);
1847 if (res.restartOneDbNode(nodes[1], false, true, true))
1848 return NDBT_FAILED;
1849
1850 if (res.waitNodesNoStart(nodes+1, 1))
1851 return NDBT_FAILED;
1852
1853 if (res.startNodes(nodes+1, 1))
1854 return NDBT_FAILED;
1855
1856 if (res.waitClusterStarted())
1857 return NDBT_FAILED;
1858
1859 nodes[1] = res.getNextMasterNodeId(nodes[0]);
1860 }
1861
1862 ndbout_c("nodes[0]: %u nodes[1]: %u", nodes[0], nodes[1]);
1863
1864 int val = DumpStateOrd::DihMinTimeBetweenLCP;
1865 if (res.dumpStateAllNodes(&val, 1))
1866 return NDBT_FAILED;
1867
1868 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1869 if (res.dumpStateAllNodes(val2, 2))
1870 return NDBT_FAILED;
1871
1872 if (res.insertErrorInAllNodes(932))
1873 return NDBT_FAILED;
1874
1875 if (res.insertErrorInNode(nodes[1], 7192))
1876 return NDBT_FAILED;
1877
1878 if (res.insertErrorInNode(nodes[0], 7191))
1879 return NDBT_FAILED;
1880
1881 if (res.waitClusterNoStart())
1882 return NDBT_FAILED;
1883
1884 if (res.startAll())
1885 return NDBT_FAILED;
1886
1887 if (res.waitClusterStarted())
1888 return NDBT_FAILED;
1889
1890 if (res.restartOneDbNode(nodes[1], false, false, true))
1891 return NDBT_FAILED;
1892
1893 if (res.waitClusterStarted())
1894 return NDBT_FAILED;
1895
1896 return NDBT_OK;
1897 }
1898
1899 int
runBug32160(NDBT_Context * ctx,NDBT_Step * step)1900 runBug32160(NDBT_Context* ctx, NDBT_Step* step)
1901 {
1902 int result = NDBT_OK;
1903 int loops = ctx->getNumLoops();
1904 int records = ctx->getNumRecords();
1905 Ndb* pNdb = GETNDB(step);
1906 NdbRestarter res;
1907
1908 if (res.getNumDbNodes() < 2)
1909 {
1910 return NDBT_OK;
1911 }
1912
1913 int master = res.getMasterNodeId();
1914 int next = res.getNextMasterNodeId(master);
1915
1916 if (res.insertErrorInNode(next, 7194))
1917 {
1918 return NDBT_FAILED;
1919 }
1920
1921 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1922 if (res.dumpStateOneNode(master, val2, 2))
1923 return NDBT_FAILED;
1924
1925 if (res.insertErrorInNode(master, 7193))
1926 return NDBT_FAILED;
1927
1928 int val3[] = { 7099 };
1929 if (res.dumpStateOneNode(master, val3, 1))
1930 return NDBT_FAILED;
1931
1932 if (res.waitNodesNoStart(&master, 1))
1933 return NDBT_FAILED;
1934
1935 if (res.startNodes(&master, 1))
1936 return NDBT_FAILED;
1937
1938 if (res.waitClusterStarted())
1939 return NDBT_FAILED;
1940
1941 return NDBT_OK;
1942 }
1943
1944 int
runBug34216(NDBT_Context * ctx,NDBT_Step * step)1945 runBug34216(NDBT_Context* ctx, NDBT_Step* step)
1946 {
1947 int result = NDBT_OK;
1948 int loops = ctx->getNumLoops();
1949 NdbRestarter restarter;
1950 int i = 0;
1951 int lastId = 0;
1952 HugoOperations hugoOps(*ctx->getTab());
1953 int records = ctx->getNumRecords();
1954 Ndb* pNdb = GETNDB(step);
1955
1956 if (restarter.getNumDbNodes() < 2)
1957 {
1958 ctx->stopTest();
1959 return NDBT_OK;
1960 }
1961
1962 if(restarter.waitClusterStarted() != 0){
1963 g_err << "Cluster failed to start" << endl;
1964 return NDBT_FAILED;
1965 }
1966
1967 char buf[100];
1968 const char * off = NdbEnv_GetEnv("NDB_ERR_OFFSET", buf, sizeof(buf));
1969 int offset = off ? atoi(off) : 0;
1970
1971 while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped())
1972 {
1973 int id = lastId % restarter.getNumDbNodes();
1974 int nodeId = restarter.getDbNodeId(id);
1975 int err = 5048 + ((i+offset) % 2);
1976
1977 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1978
1979 if(hugoOps.startTransaction(pNdb) != 0)
1980 goto err;
1981
1982 nodeId = hugoOps.getTransaction()->getConnectedNodeId();
1983 ndbout << "Restart node " << nodeId << " " << err <<endl;
1984
1985 if (restarter.dumpStateOneNode(nodeId, val2, 2))
1986 return NDBT_FAILED;
1987
1988 if(restarter.insertErrorInNode(nodeId, err) != 0){
1989 g_err << "Failed to restartNextDbNode" << endl;
1990 result = NDBT_FAILED;
1991 break;
1992 }
1993
1994 if (restarter.insertErrorInNode(nodeId, 8057) != 0)
1995 {
1996 g_err << "Failed to insert error 8057" << endl;
1997 result = NDBT_FAILED;
1998 break;
1999 }
2000
2001 int rows = 10;
2002 int batch = 1;
2003 int row = (records - rows) ? rand() % (records - rows) : 0;
2004
2005 if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
2006 goto err;
2007
2008 for (int l = 1; l<5; l++)
2009 {
2010 if (hugoOps.execute_NoCommit(pNdb) != 0)
2011 goto err;
2012
2013 if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
2014 goto err;
2015 }
2016
2017 hugoOps.execute_Commit(pNdb);
2018 hugoOps.closeTransaction(pNdb);
2019
2020 if (restarter.waitNodesNoStart(&nodeId, 1))
2021 {
2022 g_err << "Failed to waitNodesNoStart" << endl;
2023 result = NDBT_FAILED;
2024 break;
2025 }
2026
2027 if (restarter.startNodes(&nodeId, 1))
2028 {
2029 g_err << "Failed to startNodes" << endl;
2030 result = NDBT_FAILED;
2031 break;
2032 }
2033
2034 if(restarter.waitClusterStarted() != 0){
2035 g_err << "Cluster failed to start" << endl;
2036 result = NDBT_FAILED;
2037 break;
2038 }
2039
2040 lastId++;
2041 i++;
2042 }
2043
2044 ctx->stopTest();
2045
2046 return result;
2047 err:
2048 return NDBT_FAILED;
2049 }
2050
2051
2052 int
runNF_commit(NDBT_Context * ctx,NDBT_Step * step)2053 runNF_commit(NDBT_Context* ctx, NDBT_Step* step)
2054 {
2055 int result = NDBT_OK;
2056 int loops = ctx->getNumLoops();
2057 NdbRestarter restarter;
2058 if (restarter.getNumDbNodes() < 2)
2059 {
2060 ctx->stopTest();
2061 return NDBT_OK;
2062 }
2063
2064 if(restarter.waitClusterStarted() != 0){
2065 g_err << "Cluster failed to start" << endl;
2066 return NDBT_FAILED;
2067 }
2068
2069 int i = 0;
2070 while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped())
2071 {
2072 int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
2073 int err = 5048;
2074
2075 ndbout << "Restart node " << nodeId << " " << err <<endl;
2076
2077 int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2078 if (restarter.dumpStateOneNode(nodeId, val2, 2))
2079 return NDBT_FAILED;
2080
2081 if(restarter.insertErrorInNode(nodeId, err) != 0){
2082 g_err << "Failed to restartNextDbNode" << endl;
2083 result = NDBT_FAILED;
2084 break;
2085 }
2086
2087 if (restarter.waitNodesNoStart(&nodeId, 1))
2088 {
2089 g_err << "Failed to waitNodesNoStart" << endl;
2090 result = NDBT_FAILED;
2091 break;
2092 }
2093
2094 if (restarter.startNodes(&nodeId, 1))
2095 {
2096 g_err << "Failed to startNodes" << endl;
2097 result = NDBT_FAILED;
2098 break;
2099 }
2100
2101 if(restarter.waitClusterStarted() != 0){
2102 g_err << "Cluster failed to start" << endl;
2103 result = NDBT_FAILED;
2104 break;
2105 }
2106
2107 i++;
2108 }
2109
2110 ctx->stopTest();
2111
2112 return result;
2113 }
2114
2115
2116 NDBT_TESTSUITE(testNodeRestart);
2117 TESTCASE("NoLoad",
2118 "Test that one node at a time can be stopped and then restarted "\
2119 "when there are no load on the system. Do this loop number of times"){
2120 INITIALIZER(runCheckAllNodesStarted);
2121 INITIALIZER(runLoadTable);
2122 STEP(runRestarter);
2123 FINALIZER(runClearTable);
2124 }
2125 TESTCASE("PkRead",
2126 "Test that one node at a time can be stopped and then restarted "\
2127 "perform pk read while restarting. Do this loop number of times"){
2128 TC_PROPERTY("ReadLockMode", NdbOperation::LM_Read);
2129 INITIALIZER(runCheckAllNodesStarted);
2130 INITIALIZER(runLoadTable);
2131 STEP(runRestarter);
2132 STEP(runPkReadUntilStopped);
2133 FINALIZER(runClearTable);
2134 }
2135 TESTCASE("PkReadCommitted",
2136 "Test that one node at a time can be stopped and then restarted "\
2137 "perform pk read while restarting. Do this loop number of times"){
2138 TC_PROPERTY("ReadLockMode", NdbOperation::LM_CommittedRead);
2139 INITIALIZER(runCheckAllNodesStarted);
2140 INITIALIZER(runLoadTable);
2141 STEP(runRestarter);
2142 STEP(runPkReadUntilStopped);
2143 FINALIZER(runClearTable);
2144 }
2145 TESTCASE("MixedPkRead",
2146 "Test that one node at a time can be stopped and then restarted "\
2147 "perform pk read while restarting. Do this loop number of times"){
2148 TC_PROPERTY("ReadLockMode", -1);
2149 INITIALIZER(runCheckAllNodesStarted);
2150 INITIALIZER(runLoadTable);
2151 STEP(runRestarter);
2152 STEP(runPkReadUntilStopped);
2153 FINALIZER(runClearTable);
2154 }
2155 TESTCASE("PkReadPkUpdate",
2156 "Test that one node at a time can be stopped and then restarted "\
2157 "perform pk read and pk update while restarting. Do this loop number of times"){
2158 TC_PROPERTY("ReadLockMode", NdbOperation::LM_Read);
2159 INITIALIZER(runCheckAllNodesStarted);
2160 INITIALIZER(runLoadTable);
2161 STEP(runRestarter);
2162 STEP(runPkReadUntilStopped);
2163 STEP(runPkUpdateUntilStopped);
2164 STEP(runPkReadPkUpdateUntilStopped);
2165 STEP(runPkReadUntilStopped);
2166 STEP(runPkUpdateUntilStopped);
2167 STEP(runPkReadPkUpdateUntilStopped);
2168 FINALIZER(runClearTable);
2169 }
2170 TESTCASE("MixedPkReadPkUpdate",
2171 "Test that one node at a time can be stopped and then restarted "\
2172 "perform pk read and pk update while restarting. Do this loop number of times"){
2173 TC_PROPERTY("ReadLockMode", -1);
2174 INITIALIZER(runCheckAllNodesStarted);
2175 INITIALIZER(runLoadTable);
2176 STEP(runRestarter);
2177 STEP(runPkReadUntilStopped);
2178 STEP(runPkUpdateUntilStopped);
2179 STEP(runPkReadPkUpdateUntilStopped);
2180 STEP(runPkReadUntilStopped);
2181 STEP(runPkUpdateUntilStopped);
2182 STEP(runPkReadPkUpdateUntilStopped);
2183 FINALIZER(runClearTable);
2184 }
2185 TESTCASE("ReadUpdateScan",
2186 "Test that one node at a time can be stopped and then restarted "\
2187 "perform pk read, pk update and scan reads while restarting. Do this loop number of times"){
2188 INITIALIZER(runCheckAllNodesStarted);
2189 INITIALIZER(runLoadTable);
2190 STEP(runRestarter);
2191 STEP(runPkReadUntilStopped);
2192 STEP(runPkUpdateUntilStopped);
2193 STEP(runPkReadPkUpdateUntilStopped);
2194 STEP(runScanReadUntilStopped);
2195 STEP(runScanUpdateUntilStopped);
2196 FINALIZER(runClearTable);
2197 }
2198 TESTCASE("MixedReadUpdateScan",
2199 "Test that one node at a time can be stopped and then restarted "\
2200 "perform pk read, pk update and scan reads while restarting. Do this loop number of times"){
2201 TC_PROPERTY("ReadLockMode", -1);
2202 INITIALIZER(runCheckAllNodesStarted);
2203 INITIALIZER(runLoadTable);
2204 STEP(runRestarter);
2205 STEP(runPkReadUntilStopped);
2206 STEP(runPkUpdateUntilStopped);
2207 STEP(runPkReadPkUpdateUntilStopped);
2208 STEP(runScanReadUntilStopped);
2209 STEP(runScanUpdateUntilStopped);
2210 FINALIZER(runClearTable);
2211 }
2212 TESTCASE("Terror",
2213 "Test that one node at a time can be stopped and then restarted "\
2214 "perform all kind of transactions while restarting. Do this loop number of times"){
2215 INITIALIZER(runCheckAllNodesStarted);
2216 INITIALIZER(runLoadTable);
2217 STEP(runRestarter);
2218 STEP(runPkReadUntilStopped);
2219 STEP(runPkUpdateUntilStopped);
2220 STEP(runScanReadUntilStopped);
2221 STEP(runScanUpdateUntilStopped);
2222 FINALIZER(runClearTable);
2223 }
2224 TESTCASE("FullDb",
2225 "Test that one node at a time can be stopped and then restarted "\
2226 "when db is full. Do this loop number of times"){
2227 INITIALIZER(runCheckAllNodesStarted);
2228 INITIALIZER(runFillTable);
2229 STEP(runRestarter);
2230 FINALIZER(runClearTable);
2231 }
2232 TESTCASE("RestartRandomNode",
2233 "Test that we can execute the restart RestartRandomNode loop\n"\
2234 "number of times"){
2235 INITIALIZER(runCheckAllNodesStarted);
2236 INITIALIZER(runLoadTable);
2237 STEP(runRestarts);
2238 FINALIZER(runScanReadVerify);
2239 FINALIZER(runClearTable);
2240 }
2241 TESTCASE("RestartRandomNodeError",
2242 "Test that we can execute the restart RestartRandomNodeError loop\n"\
2243 "number of times"){
2244 INITIALIZER(runCheckAllNodesStarted);
2245 INITIALIZER(runLoadTable);
2246 STEP(runRestarts);
2247 FINALIZER(runScanReadVerify);
2248 FINALIZER(runClearTable);
2249 }
2250 TESTCASE("RestartRandomNodeInitial",
2251 "Test that we can execute the restart RestartRandomNodeInitial loop\n"\
2252 "number of times"){
2253 INITIALIZER(runCheckAllNodesStarted);
2254 INITIALIZER(runLoadTable);
2255 STEP(runRestarts);
2256 FINALIZER(runScanReadVerify);
2257 FINALIZER(runClearTable);
2258 }
2259 TESTCASE("RestartNFDuringNR",
2260 "Test that we can execute the restart RestartNFDuringNR loop\n"\
2261 "number of times"){
2262 INITIALIZER(runCheckAllNodesStarted);
2263 INITIALIZER(runLoadTable);
2264 STEP(runRestarts);
2265 STEP(runPkUpdateUntilStopped);
2266 STEP(runScanUpdateUntilStopped);
2267 FINALIZER(runScanReadVerify);
2268 FINALIZER(runClearTable);
2269 }
2270 TESTCASE("RestartMasterNodeError",
2271 "Test that we can execute the restart RestartMasterNodeError loop\n"\
2272 "number of times"){
2273 INITIALIZER(runCheckAllNodesStarted);
2274 INITIALIZER(runLoadTable);
2275 STEP(runRestarts);
2276 FINALIZER(runScanReadVerify);
2277 FINALIZER(runClearTable);
2278 }
2279
2280 TESTCASE("TwoNodeFailure",
2281 "Test that we can execute the restart TwoNodeFailure\n"\
2282 "(which is a multiple node failure restart) loop\n"\
2283 "number of times"){
2284 INITIALIZER(runCheckAllNodesStarted);
2285 INITIALIZER(runLoadTable);
2286 STEP(runRestarts);
2287 FINALIZER(runScanReadVerify);
2288 FINALIZER(runClearTable);
2289 }
2290 TESTCASE("TwoMasterNodeFailure",
2291 "Test that we can execute the restart TwoMasterNodeFailure\n"\
2292 "(which is a multiple node failure restart) loop\n"\
2293 "number of times"){
2294 INITIALIZER(runCheckAllNodesStarted);
2295 INITIALIZER(runLoadTable);
2296 STEP(runRestarts);
2297 FINALIZER(runScanReadVerify);
2298 FINALIZER(runClearTable);
2299 }
2300 TESTCASE("FiftyPercentFail",
2301 "Test that we can execute the restart FiftyPercentFail\n"\
2302 "(which is a multiple node failure restart) loop\n"\
2303 "number of times"){
2304 INITIALIZER(runCheckAllNodesStarted);
2305 INITIALIZER(runLoadTable);
2306 STEP(runRestarts);
2307 FINALIZER(runScanReadVerify);
2308 FINALIZER(runClearTable);
2309 }
2310 TESTCASE("RestartAllNodes",
2311 "Test that we can execute the restart RestartAllNodes\n"\
2312 "(which is a system restart) loop\n"\
2313 "number of times"){
2314 INITIALIZER(runCheckAllNodesStarted);
2315 INITIALIZER(runLoadTable);
2316 STEP(runRestarts);
2317 FINALIZER(runScanReadVerify);
2318 FINALIZER(runClearTable);
2319 }
2320 TESTCASE("RestartAllNodesAbort",
2321 "Test that we can execute the restart RestartAllNodesAbort\n"\
2322 "(which is a system restart) loop\n"\
2323 "number of times"){
2324 INITIALIZER(runCheckAllNodesStarted);
2325 INITIALIZER(runLoadTable);
2326 STEP(runRestarts);
2327 FINALIZER(runScanReadVerify);
2328 FINALIZER(runClearTable);
2329 }
2330 TESTCASE("RestartAllNodesError9999",
2331 "Test that we can execute the restart RestartAllNodesError9999\n"\
2332 "(which is a system restart) loop\n"\
2333 "number of times"){
2334 INITIALIZER(runCheckAllNodesStarted);
2335 INITIALIZER(runLoadTable);
2336 STEP(runRestarts);
2337 FINALIZER(runScanReadVerify);
2338 FINALIZER(runClearTable);
2339 }
2340 TESTCASE("FiftyPercentStopAndWait",
2341 "Test that we can execute the restart FiftyPercentStopAndWait\n"\
2342 "(which is a system restart) loop\n"\
2343 "number of times"){
2344 INITIALIZER(runCheckAllNodesStarted);
2345 INITIALIZER(runLoadTable);
2346 STEP(runRestarts);
2347 FINALIZER(runScanReadVerify);
2348 FINALIZER(runClearTable);
2349 }
2350 TESTCASE("RestartNodeDuringLCP",
2351 "Test that we can execute the restart RestartRandomNode loop\n"\
2352 "number of times"){
2353 INITIALIZER(runCheckAllNodesStarted);
2354 INITIALIZER(runLoadTable);
2355 STEP(runRestarts);
2356 STEP(runPkUpdateUntilStopped);
2357 STEP(runScanUpdateUntilStopped);
2358 FINALIZER(runScanReadVerify);
2359 FINALIZER(runClearTable);
2360 }
2361 TESTCASE("StopOnError",
2362 "Test StopOnError. A node that has StopOnError set to false "\
2363 "should restart automatically when an error occurs"){
2364 INITIALIZER(runCheckAllNodesStarted);
2365 INITIALIZER(runLoadTable);
2366 STEP(runRestarts);
2367 FINALIZER(runScanReadVerify);
2368 FINALIZER(runClearTable);
2369 }
2370 TESTCASE("CommittedRead",
2371 "Test committed read"){
2372 INITIALIZER(runLoadTable);
2373 STEP(runDirtyRead);
2374 FINALIZER(runClearTable);
2375 }
2376 TESTCASE("LateCommit",
2377 "Test commit after node failure"){
2378 INITIALIZER(runLoadTable);
2379 STEP(runLateCommit);
2380 FINALIZER(runClearTable);
2381 }
2382 TESTCASE("Bug15587",
2383 "Test bug with NF during NR"){
2384 INITIALIZER(runLoadTable);
2385 STEP(runScanUpdateUntilStopped);
2386 STEP(runBug15587);
2387 FINALIZER(runClearTable);
2388 }
2389 TESTCASE("Bug15632",
2390 "Test bug with NF during NR"){
2391 INITIALIZER(runLoadTable);
2392 STEP(runBug15632);
2393 FINALIZER(runClearTable);
2394 }
2395 TESTCASE("Bug15685",
2396 "Test bug with NF during abort"){
2397 STEP(runBug15685);
2398 FINALIZER(runClearTable);
2399 }
2400 TESTCASE("Bug16772",
2401 "Test bug with restarting before NF handling is complete"){
2402 STEP(runBug16772);
2403 }
2404 TESTCASE("Bug18414",
2405 "Test bug with NF during NR"){
2406 INITIALIZER(runLoadTable);
2407 STEP(runBug18414);
2408 FINALIZER(runClearTable);
2409 }
2410 TESTCASE("Bug18612",
2411 "Test bug with partitioned clusters"){
2412 INITIALIZER(runLoadTable);
2413 STEP(runBug18612);
2414 FINALIZER(runClearTable);
2415 }
2416 TESTCASE("Bug18612SR",
2417 "Test bug with partitioned clusters"){
2418 INITIALIZER(runLoadTable);
2419 STEP(runBug18612SR);
2420 FINALIZER(runClearTable);
2421 }
2422 TESTCASE("Bug20185",
2423 ""){
2424 INITIALIZER(runLoadTable);
2425 STEP(runBug20185);
2426 FINALIZER(runClearTable);
2427 }
2428 TESTCASE("Bug24543", "")
2429 {
2430 INITIALIZER(runBug24543);
2431 }
2432 TESTCASE("Bug21271",
2433 ""){
2434 INITIALIZER(runLoadTable);
2435 STEP(runBug21271);
2436 STEP(runPkUpdateUntilStopped);
2437 FINALIZER(runClearTable);
2438 }
2439 TESTCASE("Bug31525", ""){
2440 INITIALIZER(runBug31525);
2441 }
2442 TESTCASE("Bug24717", ""){
2443 INITIALIZER(runBug24717);
2444 }
2445 TESTCASE("Bug25364", ""){
2446 INITIALIZER(runBug25364);
2447 }
2448 TESTCASE("Bug25468", ""){
2449 INITIALIZER(runBug25468);
2450 }
2451 TESTCASE("Bug25554", ""){
2452 INITIALIZER(runBug25554);
2453 }
2454 TESTCASE("Bug25984", ""){
2455 INITIALIZER(runBug25984);
2456 }
2457 TESTCASE("Bug26457", ""){
2458 INITIALIZER(runBug26457);
2459 }
2460 TESTCASE("Bug26481", ""){
2461 INITIALIZER(runBug26481);
2462 }
2463 TESTCASE("Bug26450", ""){
2464 INITIALIZER(runLoadTable);
2465 INITIALIZER(runBug26450);
2466 }
2467 TESTCASE("Bug27003", ""){
2468 INITIALIZER(runBug27003);
2469 }
2470 TESTCASE("Bug27283", ""){
2471 INITIALIZER(runBug27283);
2472 }
2473 TESTCASE("Bug27466", ""){
2474 INITIALIZER(runBug27466);
2475 }
2476 TESTCASE("Bug28023", ""){
2477 INITIALIZER(runBug28023);
2478 }
2479 TESTCASE("Bug28717", ""){
2480 INITIALIZER(runBug28717);
2481 }
2482 TESTCASE("Bug29364", ""){
2483 INITIALIZER(runBug29364);
2484 }
2485 TESTCASE("Bug32160", ""){
2486 INITIALIZER(runBug32160);
2487 }
2488 TESTCASE("Bug34216", ""){
2489 INITIALIZER(runCheckAllNodesStarted);
2490 INITIALIZER(runLoadTable);
2491 STEP(runBug34216);
2492 FINALIZER(runClearTable);
2493 }
2494 TESTCASE("mixedmultiop", ""){
2495 TC_PROPERTY("MULTI_OP", 5);
2496 INITIALIZER(runCheckAllNodesStarted);
2497 INITIALIZER(runLoadTable);
2498 STEP(runNF_commit);
2499 STEP(runPkUpdateUntilStopped);
2500 STEP(runPkUpdateUntilStopped);
2501 FINALIZER(runClearTable);
2502 }
2503 NDBT_TESTSUITE_END(testNodeRestart);
2504
main(int argc,const char ** argv)2505 int main(int argc, const char** argv){
2506 ndb_init();
2507 #if 0
2508 // It might be interesting to have longer defaults for num
2509 // loops in this test
2510 // Just performing 100 node restarts would not be enough?
2511 // We can have initialisers in the NDBT_Testcase class like
2512 // this...
2513 testNodeRestart.setDefaultLoops(1000);
2514 #endif
2515 return testNodeRestart.execute(argc, argv);
2516 }
2517
2518