1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 1996-2018. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20 21%% 22-module(mnesia_recovery_test). 23-author('hakan@erix.ericsson.se'). 24 25-export([init_per_testcase/2, end_per_testcase/2, 26 init_per_group/2, end_per_group/2, 27 all/0, groups/0]). 28 29-export([coord_dies/1, after_full_disc_partition/1, 30 disc_less/1, garb_decision/1, 31 system_upgrade/1, 32 delete_during_start/1, 33 no_master_2/1, no_master_3/1, one_master_2/1, one_master_3/1, 34 two_master_2/1, two_master_3/1, all_master_2/1, 35 all_master_3/1, 36 dirty_read_during_down/1, trans_read_during_down/1, 37 mnesia_down_during_startup_disk_ram/1, 38 mnesia_down_during_startup_init_ram/1, 39 mnesia_down_during_startup_init_disc/1, 40 mnesia_down_during_startup_init_disc_only/1, 41 mnesia_down_during_startup_tm_ram/1, 42 mnesia_down_during_startup_tm_disc/1, 43 mnesia_down_during_startup_tm_disc_only/1, 44 with_checkpoint_same/1, with_checkpoint_other/1, 45 explicit_stop_during_snmp/1, 46 sym_trans_before_commit_kill_coord_node/1, 47 sym_trans_before_commit_kill_coord_pid/1, 48 sym_trans_before_commit_kill_part_after_ask/1, 49 sym_trans_before_commit_kill_part_before_ask/1, 50 sym_trans_after_commit_kill_coord_node/1, 51 sym_trans_after_commit_kill_coord_pid/1, 52 sym_trans_after_commit_kill_part_after_ask/1, 53 sym_trans_after_commit_kill_part_do_commit_pre/1, 54 sym_trans_after_commit_kill_part_do_commit_post/1, 55 sync_dirty_pre_kill_part/1, 56 sync_dirty_pre_kill_coord_node/1, 57 sync_dirty_pre_kill_coord_pid/1, 58 sync_dirty_post_kill_part/1, 59 sync_dirty_post_kill_coord_node/1, 60 sync_dirty_post_kill_coord_pid/1, 61 async_dirty_pre_kill_part/1, 62 async_dirty_pre_kill_coord_node/1, 63 async_dirty_pre_kill_coord_pid/1, 64 async_dirty_post_kill_part/1, 65 async_dirty_post_kill_coord_node/1, 66 async_dirty_post_kill_coord_pid/1, 67 asymtrans_part_ask/1, 68 asymtrans_part_commit_vote/1, 69 asymtrans_part_pre_commit/1, 70 asymtrans_part_log_commit/1, 71 asymtrans_part_do_commit/1, 72 asymtrans_coord_got_votes/1, 73 asymtrans_coord_pid_got_votes/1, 74 asymtrans_coord_log_commit_rec/1, 75 asymtrans_coord_pid_log_commit_rec/1, 76 asymtrans_coord_log_commit_dec/1, 77 asymtrans_coord_pid_log_commit_dec/1, 78 asymtrans_coord_rec_acc_pre_commit_log_commit/1, 79 asymtrans_coord_pid_rec_acc_pre_commit_log_commit/1, 80 asymtrans_coord_rec_acc_pre_commit_done_commit/1, 81 asymtrans_coord_pid_rec_acc_pre_commit_done_commit/1, 82 after_corrupt_files_decision_log_head/1, 83 after_corrupt_files_decision_log_tail/1, 84 after_corrupt_files_latest_log_head/1, 85 after_corrupt_files_latest_log_tail/1, 86 after_corrupt_files_table_dat_head/1, 87 after_corrupt_files_table_dat_tail/1, 88 after_corrupt_files_schema_dat_head/1, 89 after_corrupt_files_schema_dat_tail/1]). 90 91-export([reader/2, check/0, get_all_retainers/1, 92 verify_data/2, verify_where2read/1, 93 do_trans_loop/2, 94 start_stop/3, do_sym_trans/2, do_sync_dirty/2, do_async_dirty/2, 95 do_asym_trans/2, garb_handler/1, mymnesia_start/1 96 ]). 97 98 99-include("mnesia_test_lib.hrl"). 100-include_lib("kernel/include/file.hrl"). 101 102init_per_testcase(Func, Conf) -> 103 mnesia_test_lib:init_per_testcase(Func, Conf). 104 105end_per_testcase(Func, Conf) -> 106 mnesia_test_lib:end_per_testcase(Func, Conf). 107 108-define(receive_messages(Msgs), receive_messages(Msgs, ?FILE, ?LINE)). 109 110% First Some debug logging 111-define(dgb, true). 112-ifdef(dgb). 113-define(dl(X, Y), ?verbose("**TRACING: " ++ X ++ "**~n", Y)). 114-else. 115-define(dl(X, Y), ok). 116-endif. 117 118%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 119all() -> 120 [{group, mnesia_down}, {group, explicit_stop}, 121 coord_dies, {group, schema_trans}, {group, async_dirty}, 122 {group, sync_dirty}, {group, sym_trans}, 123 {group, asym_trans}, after_full_disc_partition, 124 {group, after_corrupt_files}, disc_less, garb_decision, 125 system_upgrade]. 126 127groups() -> 128 [{schema_trans, [], 129 [{mnesia_schema_recovery_test, all}]}, 130 {mnesia_down, [], 131 [{group, mnesia_down_during_startup}, 132 {group, master_node_tests}, {group, read_during_down}, 133 {group, with_checkpoint}, delete_during_start]}, 134 {master_node_tests, [], 135 [no_master_2, no_master_3, one_master_2, one_master_3, 136 two_master_2, two_master_3, all_master_2, 137 all_master_3]}, 138 {read_during_down, [], 139 [dirty_read_during_down, trans_read_during_down]}, 140 {mnesia_down_during_startup, [], 141 [mnesia_down_during_startup_disk_ram, 142 mnesia_down_during_startup_init_ram, 143 mnesia_down_during_startup_init_disc, 144 mnesia_down_during_startup_init_disc_only, 145 mnesia_down_during_startup_tm_ram, 146 mnesia_down_during_startup_tm_disc, 147 mnesia_down_during_startup_tm_disc_only]}, 148 {with_checkpoint, [], 149 [with_checkpoint_same, with_checkpoint_other]}, 150 {explicit_stop, [], [explicit_stop_during_snmp]}, 151 {sym_trans, [], 152 [sym_trans_before_commit_kill_coord_node, 153 sym_trans_before_commit_kill_coord_pid, 154 sym_trans_before_commit_kill_part_after_ask, 155 sym_trans_before_commit_kill_part_before_ask, 156 sym_trans_after_commit_kill_coord_node, 157 sym_trans_after_commit_kill_coord_pid, 158 sym_trans_after_commit_kill_part_after_ask, 159 sym_trans_after_commit_kill_part_do_commit_pre, 160 sym_trans_after_commit_kill_part_do_commit_post]}, 161 {sync_dirty, [], 162 [sync_dirty_pre_kill_part, 163 sync_dirty_pre_kill_coord_node, 164 sync_dirty_pre_kill_coord_pid, 165 sync_dirty_post_kill_part, 166 sync_dirty_post_kill_coord_node, 167 sync_dirty_post_kill_coord_pid]}, 168 {async_dirty, [], 169 [async_dirty_pre_kill_part, 170 async_dirty_pre_kill_coord_node, 171 async_dirty_pre_kill_coord_pid, 172 async_dirty_post_kill_part, 173 async_dirty_post_kill_coord_node, 174 async_dirty_post_kill_coord_pid]}, 175 {asym_trans, [], 176 [asymtrans_part_ask, 177 asymtrans_part_commit_vote, 178 asymtrans_part_pre_commit, 179 asymtrans_part_log_commit, 180 asymtrans_part_do_commit, 181 asymtrans_coord_got_votes, 182 asymtrans_coord_pid_got_votes, 183 asymtrans_coord_log_commit_rec, 184 asymtrans_coord_pid_log_commit_rec, 185 asymtrans_coord_log_commit_dec, 186 asymtrans_coord_pid_log_commit_dec, 187 asymtrans_coord_rec_acc_pre_commit_log_commit, 188 asymtrans_coord_pid_rec_acc_pre_commit_log_commit, 189 asymtrans_coord_rec_acc_pre_commit_done_commit, 190 asymtrans_coord_pid_rec_acc_pre_commit_done_commit]}, 191 {after_corrupt_files, [], 192 [after_corrupt_files_decision_log_head, 193 after_corrupt_files_decision_log_tail, 194 after_corrupt_files_latest_log_head, 195 after_corrupt_files_latest_log_tail, 196 after_corrupt_files_table_dat_head, 197 after_corrupt_files_table_dat_tail, 198 after_corrupt_files_schema_dat_head, 199 after_corrupt_files_schema_dat_tail]}]. 200 201init_per_group(_GroupName, Config) -> 202 Config. 203 204end_per_group(_GroupName, Config) -> 205 Config. 206 207tpcb_config(ReplicaType, _NodeConfig, Nodes) -> 208 [{n_branches, 5}, 209 {n_drivers_per_node, 5}, 210 {replica_nodes, Nodes}, 211 {driver_nodes, Nodes}, 212 {use_running_mnesia, true}, 213 {report_interval, infinity}, 214 {n_accounts_per_branch, 20}, 215 {replica_type, ReplicaType}]. 216 217%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 218 219 220 221no_master_2(suite) -> []; 222no_master_2(Config) when is_list(Config) -> mnesia_down_2(no, Config). 223 224no_master_3(suite) -> []; 225no_master_3(Config) when is_list(Config) -> mnesia_down_3(no, Config). 226 227one_master_2(suite) -> []; 228one_master_2(Config) when is_list(Config) -> mnesia_down_2(one, Config). 229 230one_master_3(suite) -> []; 231one_master_3(Config) when is_list(Config) -> mnesia_down_3(one, Config). 232 233two_master_2(suite) -> []; 234two_master_2(Config) when is_list(Config) -> mnesia_down_2(two, Config). 235 236two_master_3(suite) -> []; 237two_master_3(Config) when is_list(Config) -> mnesia_down_3(two, Config). 238 239all_master_2(suite) -> []; 240all_master_2(Config) when is_list(Config) -> mnesia_down_2(all, Config). 241 242all_master_3(suite) -> []; 243all_master_3(Config) when is_list(Config) -> mnesia_down_3(all, Config). 244 245mnesia_down_2(Masters, Config) -> 246 Nodes = [N1, N2] = ?acquire_nodes(2, Config), 247 ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])), 248 ?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])), 249 ?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])), 250 ?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])), 251 ?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])), 252 ?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])), 253 ?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])), 254 ?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])), 255 ?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])), 256 ?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2]}])), 257 ?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N1]}])), 258 ?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2]}])), 259 ?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N1]}])), 260 ?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2]}])), 261 ?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N1]}])), 262 263 Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, 264 tab9, tab10, tab11, tab12, tab13, tab14, tab15], 265 [?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes], 266 [insert_data(Tab, 20) || Tab <- Tabs], 267 268 VTabs = 269 case Masters of 270 no -> 271 Tabs -- [tab4, tab5]; % ram copies 272 one -> 273 ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])), 274 Tabs -- [tab1, tab4, tab5, tab10, tab12]; % ram_copies 275 two -> 276 ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])), 277 Tabs -- [tab4, tab5]; 278 all -> 279 [?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes], 280 Tabs -- [tab1, tab4, tab5, tab10, tab11, tab12, tab13] 281 end, 282 283 mnesia_test_lib:kill_mnesia([N1]), 284 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 285 286 ?match([], mnesia_test_lib:kill_mnesia([N2])), 287 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 288 289 [?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 290 [?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 291 ?verify_mnesia(Nodes, []). 292 293mnesia_down_3(Masters, Config) -> 294 Nodes = [N1, N2, N3] = ?acquire_nodes(3, Config), 295 ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])), 296 ?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])), 297 ?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])), 298 ?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])), 299 ?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])), 300 ?match({atomic, ok}, mnesia:create_table(tab16, [{ram_copies, [N3]}])), 301 ?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])), 302 ?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])), 303 ?match({atomic, ok}, mnesia:create_table(tab17, [{disc_copies, [N3]}])), 304 ?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])), 305 ?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])), 306 ?match({atomic, ok}, mnesia:create_table(tab18, [{disc_only_copies, [N3]}])), 307 ?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2, N3]}])), 308 ?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N3, N1]}])), 309 ?match({atomic, ok}, mnesia:create_table(tab19, [{ram_copies, [N3]}, {disc_copies, [N1, N2]}])), 310 ?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2, N3]}])), 311 ?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N3, N1]}])), 312 ?match({atomic, ok}, mnesia:create_table(tab20, [{ram_copies, [N3]}, {disc_only_copies, [N1, N2]}])), 313 ?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2, N3]}])), 314 ?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N3, N1]}])), 315 ?match({atomic, ok}, mnesia:create_table(tab21, [{disc_only_copies, [N3]}, {disc_copies, [N1, N2]}])), 316 317 Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, 318 tab9, tab10, tab11, tab12, tab13, tab14, tab15, 319 tab16, tab17, tab18, tab19, tab20, tab21], 320 [?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes], 321 [insert_data(Tab, 20) || Tab <- Tabs], 322 323 VTabs = 324 case Masters of 325 no -> 326 Tabs -- [tab4, tab5, tab16]; % ram copies 327 one -> 328 ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])), 329 Tabs -- [tab1, tab4, tab5, tab16, tab10, tab12]; % ram copies 330 two -> 331 ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])), 332 Tabs -- [tab4, tab5, tab16]; % ram copies 333 all -> 334 [?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes], 335 Tabs -- [tab1, tab4, tab5, tab16, tab10, 336 tab11, tab19, tab12, tab13, tab20] % ram copies 337 end, 338 339 mnesia_test_lib:kill_mnesia([N1]), 340 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 341 342 ?match([], mnesia_test_lib:kill_mnesia([N2])), 343 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 344 345 ?match([], mnesia_test_lib:kill_mnesia([N3])), 346 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 347 348 ?match([], mnesia_test_lib:kill_mnesia([N2, N1])), 349 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 350 351 ?match([], mnesia_test_lib:kill_mnesia([N2, N3])), 352 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 353 354 ?match([], mnesia_test_lib:kill_mnesia([N1, N3])), 355 ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 356 357 [?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 358 [?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 359 [?match(ok, rpc:call(N3, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 360 361 ?verify_mnesia(Nodes, []). 362 363 364 365dirty_read_during_down(suite) -> 366 []; 367dirty_read_during_down(Config) when is_list(Config) -> 368 read_during_down(dirty, Config). 369 370trans_read_during_down(suite) -> 371 []; 372trans_read_during_down(Config) when is_list(Config) -> 373 read_during_down(trans, Config). 374 375 376read_during_down(Op, Config) when is_list(Config) -> 377 Ns = [N1|TNs] = ?acquire_nodes(3, Config), 378 Tabs = [ram, disc, disco], 379 380 ?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, TNs}])), 381 ?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, TNs}])), 382 ?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, TNs}])), 383 384 %% Create some work for mnesia_controller when a node goes down 385 [{atomic, ok} = mnesia:create_table(list_to_atom("temp" ++ integer_to_list(N)), 386 [{ram_copies, Ns}]) || N <- lists:seq(1, 50)], 387 388 Write = fun(Tab) -> mnesia:write({Tab, key, val}) end, 389 ?match([ok,ok,ok], 390 [mnesia:sync_dirty(Write, [Tab]) || Tab <- Tabs]), 391 392 Readers = [spawn_link(N1, ?MODULE, reader, [Tab, Op]) || Tab <- Tabs], 393 [_|_] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs], 394 ?log("W2R ~p~n", [W2R]), 395 loop_and_kill_mnesia(10, hd(W2R), Tabs), 396 [Pid ! self() || Pid <- Readers], 397 ?match([ok, ok, ok], 398 [receive ok -> ok after 5000 -> {Pid, mnesia_lib:dist_coredump()} end 399 || Pid <- Readers]), 400 ?verify_mnesia(Ns, []). 401 402reader(Tab, OP) -> 403 Res = case OP of 404 dirty -> 405 catch mnesia:dirty_read({Tab, key}); 406 trans -> 407 Read = fun() -> mnesia:read({Tab, key}) end, 408 {_, Temp} = mnesia:transaction(Read), 409 Temp 410 end, 411 case Res of 412 [{Tab, key, val}] -> ok; 413 Else -> 414 ?error("Expected ~p Got ~p ~n", [[{Tab, key, val}], Else]), 415 erlang:error(test_failed) 416 end, 417 receive 418 Pid when is_pid(Pid) -> 419 Pid ! ok; 420 Other -> 421 io:format("Msg: ~p~n", [Other]), 422 error(Other) 423 after 50 -> 424 reader(Tab, OP) 425 end. 426 427loop_and_kill_mnesia(0, _Node, _Tabs) -> ok; 428loop_and_kill_mnesia(N, Node, Tabs) -> 429 mnesia_test_lib:kill_mnesia([Node]), 430 timer:sleep(100), 431 ?match([], mnesia_test_lib:start_mnesia([Node], Tabs)), 432 [KN | _] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs], 433 ?match([KN, KN,KN], W2R), 434 timer:sleep(100), 435 loop_and_kill_mnesia(N-1, KN, Tabs). 436 437 438mnesia_down_during_startup_disk_ram(suite) -> []; 439mnesia_down_during_startup_disk_ram(Config) when is_list(Config)-> 440 [Node1, Node2] = ?acquire_nodes(2, Config ++ 441 [{tc_timeout, timer:minutes(2)}]), 442 Tab = down_during_startup, 443 Def = [{ram_copies, [Node2]}, {disc_copies, [Node1]}], 444 445 ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 446 ?match(ok, mnesia:dirty_write({Tab, 876234, test_ok})), 447 timer:sleep(500), 448 mnesia_test_lib:kill_mnesia([Node1, Node2]), 449 timer:sleep(500), 450 mnesia_test_lib:start_mnesia([Node1, Node2], [Tab]), 451 mnesia_test_lib:kill_mnesia([Node1]), 452 timer:sleep(500), 453 ?match([], mnesia_test_lib:start_mnesia([Node1], [Tab])), 454 ?match([{Tab, 876234, test_ok}], mnesia:dirty_read({Tab,876234})), 455 ?verify_mnesia([Node1, Node2], []). 456 457mnesia_down_during_startup_init_ram(suite) -> []; 458mnesia_down_during_startup_init_ram(Config) when is_list(Config) -> 459 ?is_debug_compiled, 460 DP = {mnesia_loader, do_get_network_copy}, 461 Type = ram_copies, 462 mnesia_down_during_startup2(Config, Type, DP, self()). 463 464mnesia_down_during_startup_init_disc(suite) -> []; 465mnesia_down_during_startup_init_disc(Config) when is_list(Config) -> 466 ?is_debug_compiled, 467 DP = {mnesia_loader, do_get_network_copy}, 468 Type = disc_copies, 469 mnesia_down_during_startup2(Config, Type, DP, self()). 470 471mnesia_down_during_startup_init_disc_only(suite) -> []; 472mnesia_down_during_startup_init_disc_only(Config) when is_list(Config) -> 473 ?is_debug_compiled, 474 DP = {mnesia_loader, do_get_network_copy}, 475 Type = disc_only_copies, 476 mnesia_down_during_startup2(Config, Type, DP, self()). 477 478mnesia_down_during_startup_tm_ram(suite) -> []; 479mnesia_down_during_startup_tm_ram(Config) when is_list(Config) -> 480 ?is_debug_compiled, 481 DP = {mnesia_tm, init}, 482 Type = ram_copies, 483 mnesia_down_during_startup2(Config, Type, DP, self()). 484 485mnesia_down_during_startup_tm_disc(suite) -> []; 486mnesia_down_during_startup_tm_disc(Config) when is_list(Config) -> 487 ?is_debug_compiled, 488 DP = {mnesia_tm, init}, 489 Type = disc_copies, 490 mnesia_down_during_startup2(Config, Type, DP, self()). 491 492mnesia_down_during_startup_tm_disc_only(suite) -> []; 493mnesia_down_during_startup_tm_disc_only(Config) when is_list(Config) -> 494 ?is_debug_compiled, 495 DP = {mnesia_tm, init}, 496 Type = disc_only_copies, 497 mnesia_down_during_startup2(Config, Type, DP, self()). 498 499mnesia_down_during_startup2(Config, ReplicaType, Debug_Point, _Father) -> 500 ?log("TC~n mnesia_down_during_startup with type ~w and stops at ~w~n", 501 [ReplicaType, Debug_Point]), 502 Tpcb_tabs = [history,teller,account,branch], 503 Nodes = ?acquire_nodes(2, Config), 504 Node1 = hd(Nodes), 505 {success, [A]} = ?start_activities([Node1]), 506 TpcbConfig = tpcb_config(ReplicaType, 2, Nodes), 507 mnesia_tpcb:init(TpcbConfig), 508 A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 509 ?match_receive(timeout), 510 timer:sleep(timer:seconds(10)), % Let tpcb run for a while 511 mnesia_tpcb:stop(), 512 ?match(ok, mnesia_tpcb:verify_tabs()), 513 mnesia_test_lib:kill_mnesia([Node1]), 514 timer:sleep(timer:seconds(2)), 515 Self = self(), 516 TestFun = fun(_MnesiaEnv, _EvalEnv) -> 517 ?deactivate_debug_fun(Debug_Point), 518 Self ! fun_done, 519 spawn(mnesia_test_lib, kill_mnesia, [[Node1]]) 520 end, 521 ?activate_debug_fun(Debug_Point, TestFun, []), % Kill when debug has been reached 522 mnesia:start(), 523 Res = receive fun_done -> ok after timer:minutes(3) -> timeout end, % Wait till it's killed 524 ?match(ok, Res), 525 ?match(ok, timer:sleep(timer:seconds(2))), % Wait a while, at least till it dies; 526 ?match([], mnesia_test_lib:start_mnesia([Node1], Tpcb_tabs)), 527 ?match(ok, mnesia_tpcb:verify_tabs()), % Verify it 528 ?verify_mnesia(Nodes, []). 529 530 531 532with_checkpoint_same(suite) -> []; 533with_checkpoint_same(Config) when is_list(Config) -> 534 with_checkpoint(Config, same). 535 536with_checkpoint_other(suite) -> []; 537with_checkpoint_other(Config) when is_list(Config) -> 538 with_checkpoint(Config, other). 539 540with_checkpoint(Config, Type) when is_list(Config) -> 541 Nodes = [Node1, Node2] = ?acquire_nodes(2, Config), 542 Kill = case Type of 543 same -> %% Node1 is the one used for creating the checkpoint 544 Node1; %% and which we bring down 545 other -> 546 Node2 %% Here we bring node2 down.. 547 end, 548 549 ?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, Nodes}])), 550 ?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, Nodes}])), 551 ?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, Nodes}])), 552 Tabs = [ram, disc, disco], 553 554 ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 555 {max, mnesia:system_info(tables)}, 556 {ram_overrides_dump, true}])), 557 558 ?match([], check_retainers(sune, Nodes)), 559 560 ?match(ok, mnesia:deactivate_checkpoint(sune)), 561 ?match([], check_chkp(Nodes)), 562 563 timer:sleep(500), %% Just to help debugging the io:formats now comes in the 564 %% correct order... :-) 565 566 ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 567 {max, mnesia:system_info(tables)}, 568 {ram_overrides_dump, true}])), 569 570 [[mnesia:dirty_write({Tab,Key,Key}) || Key <- lists:seq(1,10)] || Tab <- Tabs], 571 572 mnesia_test_lib:kill_mnesia([Kill]), 573 timer:sleep(100), 574 mnesia_test_lib:start_mnesia([Kill], Tabs), 575 io:format("Mnesia on ~p started~n", [Kill]), 576 ?match([], check_retainers(sune, Nodes)), 577 ?match(ok, mnesia:deactivate_checkpoint(sune)), 578 ?match([], check_chkp(Nodes)), 579 580 Wait = fun(Loop) -> 581 timer:sleep(300), 582 sys:get_status(mnesia_monitor), 583 case lists:member(Kill, mnesia_lib:val({current, db_nodes})) of 584 true -> Loop(Loop); 585 false -> ok 586 end 587 end, 588 589 case Kill of 590 Node1 -> 591 ignore; 592 Node2 -> 593 mnesia_test_lib:kill_mnesia([Kill]), 594 Wait(Wait), 595 ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 596 {max, mnesia:system_info(tables)}, 597 {ram_overrides_dump, true}])), 598 599 [[mnesia:dirty_write({Tab,Key,Key+2}) || Key <- lists:seq(1,10)] || 600 Tab <- Tabs], 601 602 mnesia_test_lib:start_mnesia([Kill], Tabs), 603 io:format("Mnesia on ~p started ~n", [Kill]), 604 ?match([], check_retainers(sune, Nodes)), 605 ?match(ok, mnesia:deactivate_checkpoint(sune)), 606 ?match([], check_chkp(Nodes)), 607 ok 608 end, 609 ?verify_mnesia(Nodes, []). 610 611check_chkp(Nodes) -> 612 {Good, Bad} = rpc:multicall(Nodes, ?MODULE, check, []), 613 lists:flatten(Good ++ Bad). 614 615check() -> 616 [PCP] = ets:match_object(mnesia_gvar, {pending_checkpoint_pids, '_'}), 617 [PC] = ets:match_object(mnesia_gvar, {pending_checkpoints, '_'}), 618 [CPN] = ets:match_object(mnesia_gvar, {checkpoints, '_'}), 619 F = lists:filter(fun({_, []}) -> false; (_W) -> true end, 620 [PCP,PC,CPN]), 621 CPP = ets:match_object(mnesia_gvar, {{checkpoint, '_'}, '_'}), 622 Rt = ets:match_object(mnesia_gvar, {{'_', {retainer, '_'}}, '_'}), 623 F ++ CPP ++ Rt. 624 625 626check_retainers(CHP, Nodes) -> 627 {[R1,R2], []} = rpc:multicall(Nodes, ?MODULE, get_all_retainers, [CHP]), 628 (R1 -- R2) ++ (R2 -- R1). 629 630get_all_retainers(CHP) -> 631 Tabs = mnesia:system_info(local_tables), 632 Iter = fun(Tab) -> 633 {ok, Res} = 634 mnesia_checkpoint:iterate(CHP, Tab, fun(R, A) -> [R|A] end, [], 635 retainer, checkpoint), 636%% io:format("Retainer content ~w ~n", [Res]), 637 Res 638 end, 639 Elements = [Iter(Tab) || Tab <- Tabs], 640 lists:sort(lists:flatten(Elements)). 641 642delete_during_start(doc) -> 643 ["Test that tables can be delete during start, hopefully with tables" 644 " in the loader queue or soon to be"]; 645delete_during_start(suite) -> []; 646delete_during_start(Config) when is_list(Config) -> 647 [N1, N2, N3] = Nodes = ?acquire_nodes(3, Config), 648 Tabs = [list_to_atom("tab" ++ integer_to_list(I)) || I <- lists:seq(1, 30)], 649 ?match({atomic, ok}, mnesia:change_table_copy_type(schema, N2, ram_copies)), 650 ?match({atomic, ok}, mnesia:change_table_copy_type(schema, N3, ram_copies)), 651 652 [?match({atomic, ok},mnesia:create_table(Tab, [{ram_copies,Nodes}])) || Tab <- Tabs], 653 lists:foldl(fun(Tab, I) -> 654 ?match({atomic, ok}, 655 mnesia:change_table_load_order(Tab,I)), 656 I+1 657 end, 1, Tabs), 658 mnesia_test_lib:kill_mnesia([N2,N3]), 659%% timer:sleep(500), 660 ?match({[ok,ok],[]}, rpc:multicall([N2,N3], mnesia,start, 661 [[{extra_db_nodes,[N1]}, {schema, ?BACKEND}]])), 662 [Tab1,Tab2,Tab3|_] = Tabs, 663 ?match({atomic, ok}, mnesia:delete_table(Tab1)), 664 ?match({atomic, ok}, mnesia:delete_table(Tab2)), 665 666 ?log("W4T ~p~n", [rpc:multicall([N2,N3], mnesia, wait_for_tables, [[Tab1,Tab2,Tab3],1])]), 667 668 Remain = Tabs--[Tab1,Tab2], 669 ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [Remain,10000])), 670 ?match(ok, rpc:call(N3, mnesia, wait_for_tables, [Remain,10000])), 671 672 ?match(ok, rpc:call(N2, ?MODULE, verify_where2read, [Remain])), 673 ?match(ok, rpc:call(N3, ?MODULE, verify_where2read, [Remain])), 674 675 ?verify_mnesia(Nodes, []). 676 677verify_where2read([Tab|Tabs]) -> 678 true = (node() == mnesia:table_info(Tab,where_to_read)), 679 verify_where2read(Tabs); 680verify_where2read([]) -> ok. 681 682 683%%------------------------------------------------------------------------------------------- 684%% This is a bad implementation, but at least gives a indication if something is wrong 685explicit_stop_during_snmp(suite) -> []; 686explicit_stop_during_snmp(Config) when is_list(Config) -> 687 Nodes = ?acquire_nodes(2, Config), 688 [Node1, Node2] = Nodes, 689 Tab = snmp_tab, 690 Def = [{attributes, [key, value]}, 691 {snmp, [{key, integer}]}, 692 {mnesia_test_lib:storage_type(disc_copies, Config), 693 [Node1, Node2]}], 694 ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 695 ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write({Tab, 1, 1}) end)), 696 697 Do_trans_Pid1 = spawn_link(Node2, ?MODULE, do_trans_loop, [Tab, self()]), 698 Do_trans_Pid2 = spawn_link(?MODULE, do_trans_loop, [Tab, self()]), 699 Start_stop_Pid = spawn_link(?MODULE, start_stop, [Node1, 5, self()]), 700 receive 701 test_done -> 702 ok 703 after timer:minutes(5) -> 704 ?error("test case time out~n", []) 705 end, 706 ?verify_mnesia(Nodes, []), 707 exit(Do_trans_Pid1, kill), 708 exit(Do_trans_Pid2, kill), 709 exit(Start_stop_Pid, kill), 710 ok. 711 712do_trans_loop(Tab, Father) -> 713 %% Do not trap exit 714 do_trans_loop2(Tab, Father). 715do_trans_loop2(Tab, Father) -> 716 Trans = 717 fun() -> 718 [{Tab, 1, Val}] = mnesia:read({Tab, 1}), 719 mnesia:write({Tab, 1, Val + 1}) 720 end, 721 case mnesia:transaction(Trans) of 722 {atomic, ok} -> 723 timer:sleep(100), 724 do_trans_loop2(Tab, Father); 725 {aborted, {node_not_running, N}} when N == node() -> 726 timer:sleep(100), 727 do_trans_loop2(Tab, Father); 728 {aborted, {no_exists, Tab}} -> 729 timer:sleep(100), 730 do_trans_loop2(Tab, Father); 731 Else -> 732 ?error("Transaction failed: ~p ~n", [Else]), 733 io:format("INFO: ~p~n",[erlang:process_info(self())]), 734 Father ! test_done, 735 exit(shutdown) 736 end. 737 738start_stop(_Node1, 0, Father) -> 739 Father ! test_done, 740 exit(shutdown); 741start_stop(Node1, N, Father) when N > 0-> 742 timer:sleep(timer:seconds(2)), 743 ?match(stopped, rpc:call(Node1, mnesia, stop, [])), 744 timer:sleep(timer:seconds(1)), 745 ?match([], mnesia_test_lib:start_mnesia([Node1])), 746 start_stop(Node1, N-1, Father). 747 748coord_dies(suite) -> []; 749coord_dies(doc) -> [""]; 750coord_dies(Config) when is_list(Config) -> 751 Nodes = [N1, N2] = ?acquire_nodes(2, Config), 752 ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])), 753 ?match({atomic, ok}, mnesia:create_table(tab2, [{ram_copies, [N1]}])), 754 ?match({atomic, ok}, mnesia:create_table(tab3, [{ram_copies, [N2]}])), 755 Tester = self(), 756 757 U1 = fun(Tab) -> 758 [{Tab,key,Val}] = mnesia:read(Tab,key,write), 759 mnesia:write({Tab,key, Val+1}), 760 Tester ! {self(),continue}, 761 receive 762 continue -> exit(crash) 763 end 764 end, 765 U2 = fun(Tab) -> 766 [{Tab,key,Val}] = mnesia:read(Tab,key,write), 767 mnesia:write({Tab,key, Val+1}), 768 mnesia:transaction(U1, [Tab]) 769 end, 770 [mnesia:dirty_write(Tab,{Tab,key,0}) || Tab <- [tab1,tab2,tab3]], 771 Pid1 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end), 772 Pid2 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end), 773 Pid3 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end), 774 [receive {Pid,continue} -> ok end || Pid <- [Pid1,Pid2,Pid3]], 775 Pid1 ! continue, Pid2 ! continue, Pid3 ! continue, 776 ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)), 777 ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)), 778 ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)), 779 780 Pid4 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end), 781 Pid5 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end), 782 Pid6 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end), 783 erlang:monitor(process, Pid4),erlang:monitor(process, Pid5),erlang:monitor(process, Pid6), 784 785 [receive {Pid,continue} -> ok end || Pid <- [Pid4,Pid5,Pid6]], 786 exit(Pid4,crash), 787 ?match_receive({'DOWN',_,_,Pid4, _}), 788 ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)), 789 exit(Pid5,crash), 790 ?match_receive({'DOWN',_,_,Pid5, _}), 791 ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)), 792 exit(Pid6,crash), 793 ?match_receive({'DOWN',_,_,Pid6, _}), 794 ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)), 795 796 ?verify_mnesia(Nodes, []). 797 798 799%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 800 801 802%kill_after_debug_point(Config, TestCase, {Debug_node, Debug_Point}, TransFun, Tab) 803 804sym_trans_before_commit_kill_coord_node(suite) -> []; 805sym_trans_before_commit_kill_coord_node(Config) when is_list(Config) -> 806 ?is_debug_compiled, 807 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 808 [Coord, Part1, Part2] = Nodes, 809 Tab = sym_trans_before_commit_kill_coord, 810 Def = [{attributes, [key, value]}, {ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 811 kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym}}, 812 do_sym_trans, [{Tab, Def}], Nodes). 813 814sym_trans_before_commit_kill_coord_pid(suite) -> []; 815sym_trans_before_commit_kill_coord_pid(Config) when is_list(Config) -> 816 ?is_debug_compiled, 817 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 818 [Coord, Part1, Part2] = Nodes, 819 Tab = sym_trans_before_commit_kill_coord, 820 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 821 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym}}, 822 do_sym_trans, [{Tab, Def}], Nodes). 823 824sym_trans_before_commit_kill_part_after_ask(suite) -> []; 825sym_trans_before_commit_kill_part_after_ask(Config) when is_list(Config) -> 826 ?is_debug_compiled, 827 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 828 [Coord, Part1, Part2] = Nodes, 829 Tab = sym_trans_before_commit_kill_part_after_ask, 830 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 831 kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym}}, 832 do_sym_trans, [{Tab, Def}], Nodes). 833 834sym_trans_before_commit_kill_part_before_ask(suite) -> []; 835sym_trans_before_commit_kill_part_before_ask(Config) when is_list(Config) -> 836 ?is_debug_compiled, 837 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 838 [Coord, Part1, Part2] = Nodes, 839 Tab = sym_trans_before_commit_kill_part_before_ask, 840 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 841 kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}}, 842 do_sym_trans, [{Tab, Def}], Nodes). 843 844sym_trans_after_commit_kill_coord_node(suite) -> []; 845sym_trans_after_commit_kill_coord_node(Config) when is_list(Config) -> 846 ?is_debug_compiled, 847 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 848 [Coord, Part1, Part2] = Nodes, 849 Tab = sym_trans_after_commit_kill_coord, 850 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 851 kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym, post}}, 852 do_sym_trans, [{Tab, Def}], Nodes). 853 854sym_trans_after_commit_kill_coord_pid(suite) -> []; 855sym_trans_after_commit_kill_coord_pid(Config) when is_list(Config) -> 856 ?is_debug_compiled, 857 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 858 [Coord, Part1, Part2] = Nodes, 859 Tab = sym_trans_after_commit_kill_coord, 860 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 861 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym, post}}, 862 do_sym_trans, [{Tab,Def}], Nodes). 863 864sym_trans_after_commit_kill_part_after_ask(suite) -> []; 865sym_trans_after_commit_kill_part_after_ask(Config) when is_list(Config) -> 866 ?is_debug_compiled, 867 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 868 [Coord, Part1, Part2] = Nodes, 869 Tab = sym_trans_after_commit_kill_part_after_ask, 870 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 871 kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym, post}}, 872 do_sym_trans, [{Tab, Def}], Nodes). 873 874sym_trans_after_commit_kill_part_do_commit_pre(suite) -> []; 875sym_trans_after_commit_kill_part_do_commit_pre(Config) when is_list(Config) -> 876 ?is_debug_compiled, 877 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 878 [Coord, Part1, Part2] = Nodes, 879 Tab = sym_trans_after_commit_kill_part_do_commit_pre, 880 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 881 TransFun = do_sym_trans, 882 kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, pre}}, 883 TransFun, [{Tab, Def}], Nodes). 884 885sym_trans_after_commit_kill_part_do_commit_post(suite) -> []; 886sym_trans_after_commit_kill_part_do_commit_post(Config) when is_list(Config) -> 887 ?is_debug_compiled, 888 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 889 [Coord, Part1, Part2] = Nodes, 890 Tab = sym_trans_after_commit_kill_part_do_commit_post, 891 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 892 TransFun = do_sym_trans, 893 kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, post}}, 894 TransFun, [{Tab, Def}], Nodes). 895 896do_sym_trans([Tab], _Fahter) -> 897 ?dl("Starting SYM_TRANS with active debug fun ", []), 898 Trans = fun() -> 899 [{_,_,Val}] = mnesia:read({Tab, 1}), 900 mnesia:write({Tab, 1, Val+1}) 901 end, 902 Res = mnesia:transaction(Trans), 903 case Res of 904 {atomic, ok} -> ok; 905 {aborted, _Reason} -> ok; 906 Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n", 907 [Else]) 908 end, 909 ?dl("SYM_TRANSACTION done: ~p (deactiv dbgfun) ", [Res]), 910 ok. 911 912%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 913 914 915sync_dirty_pre_kill_part(suite) -> []; 916sync_dirty_pre_kill_part(Config) when is_list(Config) -> 917 ?is_debug_compiled, 918 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 919 [Coord, Part1, Part2] = Nodes, 920 Tab = sync_dirty_pre, 921 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 922 TransFun = do_sync_dirty, 923 kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, pre}}, 924 TransFun, [{Tab, Def}], Nodes). 925 926sync_dirty_pre_kill_coord_node(suite) -> []; 927sync_dirty_pre_kill_coord_node(Config) when is_list(Config) -> 928 ?is_debug_compiled, 929 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 930 [Coord, Part1, Part2] = Nodes, 931 Tab = sync_dirty_pre, 932 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 933 TransFun = do_sync_dirty, 934 kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, pre}}, 935 TransFun, [{Tab, Def}], Nodes). 936 937sync_dirty_pre_kill_coord_pid(suite) -> []; 938sync_dirty_pre_kill_coord_pid(Config) when is_list(Config) -> 939 ?is_debug_compiled, 940 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 941 [Coord, Part1, Part2] = Nodes, 942 Tab = sync_dirty_pre, 943 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 944 TransFun = do_sync_dirty, 945 kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, pre}}, 946 TransFun, [{Tab, Def}], Nodes). 947 948sync_dirty_post_kill_part(suite) -> []; 949sync_dirty_post_kill_part(Config) when is_list(Config) -> 950 ?is_debug_compiled, 951 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 952 [Coord, Part1, Part2] = Nodes, 953 Tab = sync_dirty_post, 954 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 955 TransFun = do_sync_dirty, 956 kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, post}}, 957 TransFun, [{Tab, Def}], Nodes). 958 959sync_dirty_post_kill_coord_node(suite) -> []; 960sync_dirty_post_kill_coord_node(Config) when is_list(Config) -> 961 ?is_debug_compiled, 962 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 963 [Coord, Part1, Part2] = Nodes, 964 Tab = sync_dirty_post, 965 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 966 TransFun = do_sync_dirty, 967 kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, post}}, 968 TransFun, [{Tab, Def}], Nodes). 969 970sync_dirty_post_kill_coord_pid(suite) -> []; 971sync_dirty_post_kill_coord_pid(Config) when is_list(Config) -> 972 ?is_debug_compiled, 973 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 974 [Coord, Part1, Part2] = Nodes, 975 Tab = sync_dirty_post, 976 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 977 TransFun = do_sync_dirty, 978 kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, post}}, 979 TransFun, [{Tab, Def}], Nodes). 980 981do_sync_dirty([Tab], _Father) -> 982 ?dl("Starting SYNC_DIRTY", []), 983 SYNC = fun() -> 984 [{_,_,Val}] = mnesia:read({Tab, 1}), 985 mnesia:write({Tab, 1, Val+1}) 986 end, 987 {_, Res} = ?match(ok, mnesia:sync_dirty(SYNC)), 988 ?dl("SYNC_DIRTY done: ~p ", [Res]), 989 ok. 990 991 992async_dirty_pre_kill_part(suite) -> []; 993async_dirty_pre_kill_part(Config) when is_list(Config) -> 994 ?is_debug_compiled, 995 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 996 [Coord, Part1, Part2] = Nodes, 997 Tab = async_dirty_pre, 998 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 999 TransFun = do_async_dirty, 1000 kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, pre}}, 1001 TransFun, [{Tab, Def}], Nodes). 1002 1003async_dirty_pre_kill_coord_node(suite) -> []; 1004async_dirty_pre_kill_coord_node(Config) when is_list(Config) -> 1005 ?is_debug_compiled, 1006 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1007 [Coord, Part1, Part2] = Nodes, 1008 Tab = async_dirty_pre, 1009 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 1010 TransFun = do_async_dirty, 1011 kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, pre}}, 1012 TransFun, [{Tab, Def}], Nodes). 1013 1014async_dirty_pre_kill_coord_pid(suite) -> []; 1015async_dirty_pre_kill_coord_pid(Config) when is_list(Config) -> 1016 ?is_debug_compiled, 1017 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1018 [Coord, Part1, Part2] = Nodes, 1019 Tab = async_dirty_pre, 1020 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 1021 TransFun = do_async_dirty, 1022 kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, pre}}, 1023 TransFun, [{Tab, Def}], Nodes). 1024 1025async_dirty_post_kill_part(suite) -> []; 1026async_dirty_post_kill_part(Config) when is_list(Config) -> 1027 ?is_debug_compiled, 1028 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1029 [Coord, Part1, Part2] = Nodes, 1030 Tab = async_dirty_post, 1031 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 1032 TransFun = do_async_dirty, 1033 kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, post}}, 1034 TransFun, [{Tab, Def}], Nodes). 1035 1036async_dirty_post_kill_coord_node(suite) -> []; 1037async_dirty_post_kill_coord_node(Config) when is_list(Config) -> 1038 ?is_debug_compiled, 1039 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1040 [Coord, Part1, Part2] = Nodes, 1041 Tab = async_dirty_post, 1042 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 1043 TransFun = do_async_dirty, 1044 kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, post}}, 1045 TransFun, [{Tab, Def}], Nodes). 1046 1047async_dirty_post_kill_coord_pid(suite) -> []; 1048async_dirty_post_kill_coord_pid(Config) when is_list(Config) -> 1049 ?is_debug_compiled, 1050 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1051 [Coord, Part1, Part2] = Nodes, 1052 Tab = async_dirty_post, 1053 Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 1054 TransFun = do_async_dirty, 1055 kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, post}}, 1056 TransFun, [{Tab, Def}], Nodes). 1057 1058do_async_dirty([Tab], _Fahter) -> 1059 ?dl("Starting ASYNC", []), 1060 ASYNC = fun() -> 1061 [{_,_,Val}] = mnesia:read({Tab, 1}), 1062 mnesia:write({Tab, 1, Val+1}) 1063 end, 1064 {_, Res} = ?match(ok, mnesia:async_dirty(ASYNC)), 1065 ?dl("ASYNC done: ~p ", [Res]), 1066 ok. 1067 1068%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1069 1070 1071asymtrans_part_ask(suite) -> []; 1072asymtrans_part_ask(Config) when is_list(Config) -> 1073 ?is_debug_compiled, 1074 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1075 [Coord, Part1, Part2] = Nodes, 1076 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1077 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1078 TransFun = do_asym_trans, 1079 kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}}, 1080 TransFun, [Tab1, Tab2], Nodes). 1081 1082asymtrans_part_commit_vote(suite) -> []; 1083asymtrans_part_commit_vote(Config) when is_list(Config) -> 1084 ?is_debug_compiled, 1085 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1086 [Coord, Part1, Part2] = Nodes, 1087 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1088 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1089 TransFun = do_asym_trans, 1090 kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, vote_yes}}, 1091 TransFun, [Tab1, Tab2], Nodes). 1092 1093asymtrans_part_pre_commit(suite) -> []; 1094asymtrans_part_pre_commit(Config) when is_list(Config) -> 1095 ?is_debug_compiled, 1096 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1097 [Coord, Part1, Part2] = Nodes, 1098 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1099 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1100 TransFun = do_asym_trans, 1101 kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, pre_commit}}, 1102 TransFun, [Tab1, Tab2], Nodes). 1103 1104asymtrans_part_log_commit(suite) -> []; 1105asymtrans_part_log_commit(Config) when is_list(Config) -> 1106 ?is_debug_compiled, 1107 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1108 [Coord, Part1, Part2] = Nodes, 1109 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1110 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1111 TransFun = do_asym_trans, 1112 kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, log_commit}}, 1113 TransFun, [Tab1, Tab2], Nodes). 1114 1115asymtrans_part_do_commit(suite) -> []; 1116asymtrans_part_do_commit(Config) when is_list(Config) -> 1117 ?is_debug_compiled, 1118 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1119 [Coord, Part1, Part2] = Nodes, 1120 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1121 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1122 TransFun = do_asym_trans, 1123 kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, do_commit}}, 1124 TransFun, [Tab1, Tab2], Nodes). 1125 1126asymtrans_coord_got_votes(suite) -> []; 1127asymtrans_coord_got_votes(Config) when is_list(Config) -> 1128 ?is_debug_compiled, 1129 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1130 [Coord, Part1, Part2] = Nodes, 1131 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1132 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1133 TransFun = do_asym_trans, 1134 kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_got_votes}}, 1135 TransFun, [Tab1, Tab2], Nodes). 1136 1137asymtrans_coord_pid_got_votes(suite) -> []; 1138asymtrans_coord_pid_got_votes(Config) when is_list(Config) -> 1139 ?is_debug_compiled, 1140 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1141 [Coord, Part1, Part2] = Nodes, 1142 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1143 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1144 TransFun = do_asym_trans, 1145 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_got_votes}}, 1146 TransFun, [Tab1, Tab2], Nodes). 1147 1148asymtrans_coord_log_commit_rec(suite) -> []; 1149asymtrans_coord_log_commit_rec(Config) when is_list(Config) -> 1150 ?is_debug_compiled, 1151 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1152 [Coord, Part1, Part2] = Nodes, 1153 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1154 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1155 TransFun = do_asym_trans, 1156 kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}}, 1157 TransFun, [Tab1, Tab2], Nodes). 1158 1159asymtrans_coord_pid_log_commit_rec(suite) -> []; 1160asymtrans_coord_pid_log_commit_rec(Config) when is_list(Config) -> 1161 ?is_debug_compiled, 1162 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1163 [Coord, Part1, Part2] = Nodes, 1164 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1165 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1166 TransFun = do_asym_trans, 1167 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}}, 1168 TransFun, [Tab1, Tab2], Nodes). 1169 1170asymtrans_coord_log_commit_dec(suite) -> []; 1171asymtrans_coord_log_commit_dec(Config) when is_list(Config) -> 1172 ?is_debug_compiled, 1173 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1174 [Coord, Part1, Part2] = Nodes, 1175 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1176 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1177 TransFun = do_asym_trans, 1178 kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}}, 1179 TransFun, [Tab1, Tab2], Nodes). 1180 1181asymtrans_coord_pid_log_commit_dec(suite) -> []; 1182asymtrans_coord_pid_log_commit_dec(Config) when is_list(Config) -> 1183 ?is_debug_compiled, 1184 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1185 [Coord, Part1, Part2] = Nodes, 1186 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1187 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1188 TransFun = do_asym_trans, 1189 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}}, 1190 TransFun, [Tab1, Tab2], Nodes). 1191 1192asymtrans_coord_rec_acc_pre_commit_log_commit(suite) -> []; 1193asymtrans_coord_rec_acc_pre_commit_log_commit(Config) when is_list(Config) -> 1194 ?is_debug_compiled, 1195 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1196 [Coord, Part1, Part2] = Nodes, 1197 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1198 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1199 TransFun = do_asym_trans, 1200 kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}}, 1201 TransFun, [Tab1, Tab2], Nodes). 1202 1203asymtrans_coord_pid_rec_acc_pre_commit_log_commit(suite) -> []; 1204asymtrans_coord_pid_rec_acc_pre_commit_log_commit(Config) when is_list(Config) -> 1205 ?is_debug_compiled, 1206 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1207 [Coord, Part1, Part2] = Nodes, 1208 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1209 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1210 TransFun = do_asym_trans, 1211 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}}, 1212 TransFun, [Tab1, Tab2], Nodes). 1213 1214asymtrans_coord_rec_acc_pre_commit_done_commit(suite) -> []; 1215asymtrans_coord_rec_acc_pre_commit_done_commit(Config) when is_list(Config) -> 1216 ?is_debug_compiled, 1217 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1218 [Coord, Part1, Part2] = Nodes, 1219 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1220 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1221 TransFun = do_asym_trans, 1222 kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}}, 1223 TransFun, [Tab1, Tab2], Nodes). 1224 1225asymtrans_coord_pid_rec_acc_pre_commit_done_commit(suite) -> []; 1226asymtrans_coord_pid_rec_acc_pre_commit_done_commit(Config) when is_list(Config) -> 1227 ?is_debug_compiled, 1228 Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1229 [Coord, Part1, Part2] = Nodes, 1230 Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1231 Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1232 TransFun = do_asym_trans, 1233 kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}}, 1234 TransFun, [Tab1, Tab2], Nodes). 1235 1236do_asym_trans([Tab1, Tab2 | _R], Garbhandler) -> 1237 ?dl("Starting asym trans ", []), 1238 ASym_Trans = fun() -> 1239 TidTs = {_Mod, Tid, _Store} = 1240 mnesia:get_activity_id(), 1241 ?verbose("===> asym_trans: ~w~n", [TidTs]), 1242 Garbhandler ! {trans_id, Tid}, 1243 [{_, _, Val1}] = mnesia:read({Tab1, 1}), 1244 [{_, _, Val2}] = mnesia:read({Tab2, 1}), 1245 mnesia:write({Tab1, 1, Val1+1}), 1246 mnesia:write({Tab2, 1, Val2+1}) 1247 end, 1248 Res = mnesia:transaction(ASym_Trans), 1249 case Res of 1250 {atomic, ok} -> ok; 1251 {aborted, _Reason} -> ok; 1252 _Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n", [Res]) 1253 end, 1254 ?dl("Asym trans finished with: ~p ", [Res]). 1255 1256%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1257 1258kill_after_debug_point(Kill, {DebugNode, Debug_Point}, TransFun, TabsAndDefs, Nodes) -> 1259 [Coord | _rest] = Nodes, 1260 1261 Create = fun({Tab, Def}) -> ?match({atomic, ok}, mnesia:create_table(Tab, Def)) end, 1262 lists:foreach(Create, TabsAndDefs), 1263 Tabs = [T || {T, _} <- TabsAndDefs], 1264 Write = fun(Tab) -> ?match(ok, mnesia:dirty_write({Tab, 1, 100})) end, 1265 lists:foreach(Write, Tabs), 1266 1267 Self = self(), 1268 SyncFun = fun(_Env1, _Env2) -> % Just Sync with test prog 1269 Self ! {self(), fun_in_position}, 1270 ?dl("SyncFun, sending fun_in_position ", []), 1271 receive continue -> 1272 ?dl("SyncFun received continue ",[]), 1273 ok 1274 after timer:seconds(60) -> 1275 ?error("Timeout in sync_fun on ~p~n", [node()]) 1276 end 1277 end, 1278 1279 Garb_handler = spawn_link(?MODULE, garb_handler, [[]]), 1280 1281 ?remote_activate_debug_fun(DebugNode, Debug_Point, SyncFun, []), 1282 ?dl("fun_in_position activated at ~p with ~p", [DebugNode, Debug_Point]), 1283 %% Spawn and do the transaction 1284 Pid = spawn(Coord, ?MODULE, TransFun, [Tabs, Garb_handler]), 1285 %% Wait till all the Nodes are in correct position 1286 [{StoppedPid,_}] = ?receive_messages([fun_in_position]), 1287 ?dl("Received fun_in_position; Removing the debug funs ~p", [DebugNode]), 1288 ?remote_deactivate_debug_fun(DebugNode, Debug_Point), 1289 1290 case Kill of 1291 coord_pid -> 1292 ?dl("Intentionally killing pid ~p ", [Pid]), 1293 exit(Pid, normal); 1294 Node -> 1295 mnesia_test_lib:kill_mnesia([Node]) 1296 end, 1297 1298 StoppedPid ! continue, %% Send continue, it may still be alive 1299 1300 %% Start and check that the databases are consistent 1301 ?dl("Done, Restarting and verifying result ",[]), 1302 case Kill of 1303 coord_pid -> ok; 1304 _ -> % Ok, mnesia on some node was killed restart it 1305 timer:sleep(timer:seconds(3)), %% Just let it have the time to die 1306 ?match(ok, rpc:call(Kill, mnesia, start, [[]])), 1307 ?match(ok, rpc:call(Kill, mnesia, wait_for_tables, [Tabs, 60000])) 1308 end, 1309 Trans_res = verify_tabs(Tabs, Nodes), 1310 case TransFun of 1311 do_asym_trans -> 1312 %% Verifies that decisions are garbed, only valid for asym_tran 1313 Garb_handler ! {get_tids, self()}, 1314 Tid_list = receive 1315 {tids, List} -> 1316 ?dl("Fun rec ~w", [List]), 1317 List 1318 end, 1319 garb_of_decisions(Kill, Nodes, Tid_list, Trans_res); 1320 _ -> 1321 ignore 1322 end, 1323 ?verify_mnesia(Nodes, []). 1324 1325garb_of_decisions(Kill, Nodes, Tid_list, Trans_res) -> 1326 [Coord, Part1, Part2] = Nodes, 1327 %% Check that decision log is empty on all nodes after the trans is finished 1328 verify_garb_decision_log(Nodes, Tid_list), 1329 case Trans_res of 1330 aborted -> 1331 %% Check that aborted trans have not been restarted!! 1332 ?match(1, length(Tid_list)), 1333 %% Check the transient decision logs 1334 %% A transaction should only be aborted in an early stage of 1335 %% the trans before the any Node have logged anything 1336 verify_garb_transient_logs(Nodes, Tid_list, aborted), 1337 %% And only when the coordinator are have died 1338 %% Else he would have restarted the transaction 1339 ?match(Kill, Coord); 1340 updated -> 1341 case length(Tid_list) of 1342 1 -> 1343 %% If there was only one transaction, it should be logged as 1344 %% comitted on every node! 1345 [Tid1] = Tid_list, 1346 verify_garb_transient_logs(Nodes, [Tid1], committed); 1347 2 -> 1348 %% If there is two transaction id, then the first 1349 %% TID should have been aborted and the transaction 1350 %% restarted with a new TID 1351 [Tid1, Tid2] = Tid_list, 1352 verify_garb_transient_logs(Nodes, [Tid1], aborted), 1353 %% If mnesia is killed on a node i.e Coord and Part1 than they 1354 %% won't know about the restarted trans! The rest of the nodes 1355 %% should know that the trans was committed 1356 case Kill of 1357 coord_pid -> 1358 verify_garb_transient_logs(Nodes, [Tid2], committed); 1359 Coord -> 1360 verify_garb_transient_logs([Part1, Part2], [Tid2], committed), 1361 verify_garb_transient_logs([Coord], [Tid2], not_found); 1362 Part1 -> 1363 verify_garb_transient_logs([Coord, Part2], [Tid2], committed), 1364 verify_garb_transient_logs([Part1], [Tid2], not_found) 1365 end 1366 end 1367 end. 1368 1369verify_garb_decision_log([], _Tids) -> ok; 1370verify_garb_decision_log([Node|R], Tids) -> 1371 Check = fun(Tid) -> %% Node, Tid used in debugging! 1372 ?match({{not_found, _}, Node, Tid}, 1373 {outcome(Tid, [mnesia_decision]), Node, Tid}) 1374 end, 1375 rpc:call(Node, lists, foreach, [Check, Tids]), 1376 verify_garb_decision_log(R, Tids). 1377 1378verify_garb_transient_logs([], _Tids, _) -> ok; 1379verify_garb_transient_logs([Node|R], Tids, Exp_Res) -> 1380 Check = fun(Tid) -> 1381 LatestTab = mnesia_lib:val(latest_transient_decision), 1382 PrevTabs = mnesia_lib:val(previous_transient_decisions), 1383 case outcome(Tid, [LatestTab | PrevTabs]) of 1384 {found, {_, [{_,_Tid, Exp_Res}]}} -> ok; 1385 {not_found, _} when Exp_Res == not_found -> ok; 1386 {not_found, _} when Exp_Res == aborted -> ok; 1387 Else -> ?error("Expected ~p in trans ~p on ~p got ~p~n", 1388 [Exp_Res, Tid, Node, Else]) 1389 end 1390 end, 1391 rpc:call(Node, lists, foreach, [Check, Tids]), 1392 verify_garb_transient_logs(R, Tids, Exp_Res). 1393 1394outcome(Tid, Tabs) -> 1395 outcome(Tid, Tabs, Tabs). 1396 1397outcome(Tid, [Tab | Tabs], AllTabs) -> 1398 case catch ets:lookup(Tab, Tid) of 1399 {'EXIT', _} -> 1400 outcome(Tid, Tabs, AllTabs); 1401 [] -> 1402 outcome(Tid, Tabs, AllTabs); 1403 Val -> 1404 {found, {Tab, Val}} 1405 end; 1406outcome(_Tid, [], AllTabs) -> 1407 {not_found, AllTabs}. 1408 1409 1410verify_tabs([Tab|R], Nodes) -> 1411 [_Coord, Part1, Part2 | _rest] = Nodes, 1412 Read = fun() -> mnesia:read({Tab, 1}) end, 1413 {success, A} = ?match({atomic, _}, mnesia:transaction(Read)), 1414 ?match(A, rpc:call(Part1, mnesia, transaction, [Read])), 1415 ?match(A, rpc:call(Part2, mnesia, transaction, [Read])), 1416 {atomic, [{Tab, 1, Res}]} = A, 1417 verify_tabs(R, Nodes, Res). 1418 1419verify_tabs([], _Nodes, Res) -> 1420 case Res of 1421 100 -> aborted; 1422 101 -> updated 1423 end; 1424 1425verify_tabs([Tab | Rest], Nodes, Res) -> 1426 [Coord, Part1, Part2 | _rest] = Nodes, 1427 Read = fun() -> mnesia:read({Tab, 1}) end, 1428 Exp = {atomic, [{Tab, 1, Res}]}, 1429 ?match(Exp, rpc:call(Coord, mnesia, transaction, [Read])), 1430 ?match(Exp, rpc:call(Part1, mnesia, transaction, [Read])), 1431 ?match(Exp, rpc:call(Part2, mnesia, transaction, [Read])), 1432 verify_tabs(Rest, Nodes, Res). 1433 1434%% Gather TIDS and send them to requesting process and exit! 1435garb_handler(List) -> 1436 receive 1437 {trans_id, ID} -> garb_handler([ID|List]); 1438 {get_tids, Pid} -> Pid ! {tids, lists:reverse(List)} 1439 end. 1440 1441%%%%%%%%%%%%%%%%%%%%%%% 1442receive_messages([], _File, _Line) -> []; 1443receive_messages(ListOfMsgs, File, Line) -> 1444 receive 1445 {Pid, Msg} -> 1446 case lists:member(Msg, ListOfMsgs) of 1447 false -> 1448 mnesia_test_lib:log("<>WARNING<>~n" 1449 "Received unexpected msg~n ~p ~n" 1450 "While waiting for ~p~n", 1451 [{Pid, Msg}, ListOfMsgs], File, Line), 1452 receive_messages(ListOfMsgs, File, Line); 1453 true -> 1454 ?dl("Got msg ~p from ~p ", [Msg, node(Pid)]), 1455 [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg], File, Line)] 1456 end; 1457 Else -> mnesia_test_lib:log("<>WARNING<>~n" 1458 "Recevied unexpected or bad formatted msg~n ~p ~n" 1459 "While waiting for ~p~n", 1460 [Else, ListOfMsgs], File, Line), 1461 receive_messages(ListOfMsgs, File, Line) 1462 after timer:minutes(2) -> 1463 ?error("Timeout in receive msgs while waiting for ~p~n", 1464 [ListOfMsgs]) 1465 end. 1466 1467%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1468 1469after_full_disc_partition(doc) -> 1470 ["Verify that the database does not get corrupt", 1471 "when Mnesia encounters a full disc partition"]. 1472 1473%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1474%% interrupted_fallback_start 1475%% is implemented in consistency interupted_install_fallback! 1476%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1477 1478after_corrupt_files_decision_log_head(suite) -> []; 1479after_corrupt_files_decision_log_head(Config) when is_list(Config) -> 1480 after_corrupt_files(Config, "DECISION.LOG", head, repair). 1481 1482after_corrupt_files_decision_log_tail(suite) -> []; 1483after_corrupt_files_decision_log_tail(Config) when is_list(Config) -> 1484 after_corrupt_files(Config, "DECISION.LOG", tail, repair). 1485 1486after_corrupt_files_latest_log_head(suite) -> []; 1487after_corrupt_files_latest_log_head(Config) when is_list(Config) -> 1488 after_corrupt_files(Config, "LATEST.LOG", head, repair). 1489 1490after_corrupt_files_latest_log_tail(suite) -> []; 1491after_corrupt_files_latest_log_tail(Config) when is_list(Config) -> 1492 after_corrupt_files(Config, "LATEST.LOG", tail, repair). 1493 1494after_corrupt_files_table_dat_head(suite) -> []; 1495after_corrupt_files_table_dat_head(Config) when is_list(Config) -> 1496 after_corrupt_files(Config, "rec_files.DAT", head, crash). 1497 1498after_corrupt_files_table_dat_tail(suite) -> []; 1499after_corrupt_files_table_dat_tail(Config) when is_list(Config) -> 1500 after_corrupt_files(Config, "rec_files.DAT", tail, repair). 1501 1502after_corrupt_files_schema_dat_head(suite) -> []; 1503after_corrupt_files_schema_dat_head(Config) when is_list(Config) -> 1504 after_corrupt_files(Config, "schema.DAT", head, crash). 1505 1506after_corrupt_files_schema_dat_tail(suite) -> []; 1507after_corrupt_files_schema_dat_tail(Config) when is_list(Config) -> 1508 after_corrupt_files(Config, "schema.DAT", tail, crash). 1509 1510 1511 1512%%% BUGBUG: We should also write testcase's for autorepair=false i.e. 1513%%% not the standard case! 1514after_corrupt_files(Config, File, Where, Behaviour) -> 1515 [Node] = ?acquire_nodes(1, Config ++ [{tc_timeout, timer:minutes(2)}]), 1516 Tab = rec_files, 1517 Def = [{disc_only_copies, [Node]}], 1518 ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 1519 insert_data(Tab, 100), 1520 Dir = mnesia:system_info(directory), 1521 mnesia_test_lib:kill_mnesia([Node]), 1522 timer:sleep(timer:seconds(10)), % Let dets finish whatever it does 1523 1524 DirFile = Dir ++ "/" ++ File, 1525 1526 {ok, Fd} = file:open(DirFile, read_write), 1527 {ok, FileInfo} = file:read_file_info(DirFile), 1528 case Where of 1529 head -> 1530 ?match({ok, _NewP}, file:position(Fd, {bof, 1})), 1531 ?match(ok, file:write(Fd, [255, 255, 255, 255, 255, 255, 255, 255, 254])), 1532 ok; 1533 tail -> 1534 Size = FileInfo#file_info.size, 1535 Half = Size div 2, 1536 1537 ?dl(" Size = ~p Half = ~p ", [Size, Half]), 1538 ?match({ok, _NewP}, file:position(Fd, {bof, Half})), 1539 ?match(ok, file:truncate(Fd)), 1540 ok 1541 end, 1542 ?match(ok, file:close(Fd)), 1543 1544 ?warning("++++++SOME OF THE after_corrupt* TEST CASES WILL INTENTIONALLY CRASH MNESIA+++++++~n", []), 1545 Pid = spawn_link(?MODULE, mymnesia_start, [self()]), 1546 receive 1547 {Pid, ok} -> 1548 ?match(ok, mnesia:wait_for_tables([schema, Tab], 10000)), 1549 ?match(ok, verify_data(Tab, 100)), 1550 case mnesia_monitor:get_env(auto_repair) of 1551 false -> 1552 ?error("Mnesia should have crashed in ~p ~p ~n", 1553 [File, Where]); 1554 true -> 1555 ok 1556 end, 1557 ?verify_mnesia([Node], []); 1558 {Pid, {error, ED}} -> 1559 case {mnesia_monitor:get_env(auto_repair), Behaviour} of 1560 {true, repair} -> 1561 ?error("Mnesia crashed with ~p: in ~p ~p ~n", 1562 [ED, File, Where]); 1563 _ -> %% Every other can crash! 1564 ok 1565 end, 1566 ?verify_mnesia([], [Node]); 1567 Msg -> 1568 ?error("~p ~p: Got ~p during start of Mnesia~n", 1569 [File, Where, Msg]) 1570 end. 1571 1572mymnesia_start(Tester) -> 1573 Res = mnesia:start(), 1574 unlink(Tester), 1575 Tester ! {self(), Res}. 1576 1577verify_data(_, 0) -> ok; 1578verify_data(Tab, N) -> 1579 Actual = mnesia:dirty_read({Tab, N}), 1580 Expected = [{Tab, N, N}], 1581 if 1582 Expected == Actual -> 1583 verify_data(Tab, N - 1); 1584 true -> 1585 mnesia:schema(Tab), 1586 {not_equal, node(), Expected, Actual} 1587 end. 1588 1589insert_data(_Tab, 0) -> ok; 1590insert_data(Tab, N) -> 1591 ok = mnesia:sync_dirty(fun() -> mnesia:write({Tab, N, N}) end), 1592 insert_data(Tab, N-1). 1593 1594%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1595 1596disc_less(doc) -> 1597 ["Here is a simple test case of a simple recovery of a disc less node. " 1598 "However a lot more test cases involving disc less nodes should " 1599 "be written"]; 1600disc_less(suite) -> []; 1601disc_less(Config) when is_list(Config) -> 1602 [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), 1603 case mnesia_test_lib:diskless(Config) of 1604 true -> skip; 1605 false -> 1606 ?match({atomic, ok}, mnesia:change_table_copy_type(schema, Node3, ram_copies)) 1607 end, 1608 Tab1 = disc_less1, 1609 Tab2 = disc_less2, 1610 Tab3 = disc_less3, 1611 Def1 = [{ram_copies, [Node3]}, {disc_copies, [Node1, Node2]}], 1612 Def2 = [{ram_copies, [Node3]}, {disc_copies, [Node1]}], 1613 Def3 = [{ram_copies, [Node3]}, {disc_copies, [Node2]}], 1614 ?match({atomic, ok}, mnesia:create_table(Tab1, Def1)), 1615 ?match({atomic, ok}, mnesia:create_table(Tab2, Def2)), 1616 ?match({atomic, ok}, mnesia:create_table(Tab3, Def3)), 1617 insert_data(Tab1, 100), 1618 insert_data(Tab2, 100), 1619 insert_data(Tab3, 100), 1620 1621 mnesia_test_lib:kill_mnesia([Node1, Node2]), 1622 timer:sleep(500), 1623 mnesia_test_lib:kill_mnesia([Node3]), 1624 ?match(ok, rpc:call(Node1, mnesia, start, [])), 1625 ?match(ok, rpc:call(Node2, mnesia, start, [])), 1626 1627 timer:sleep(500), 1628 ?match(ok, rpc:call(Node3, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}, {schema, ?BACKEND}]])), 1629 ?match(ok, rpc:call(Node3, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])), 1630 ?match(ok, rpc:call(Node1, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])), 1631 1632 ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab1, 100])), 1633 ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab2, 100])), 1634 ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab3, 100])), 1635 1636 1637 ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab1, 100])), 1638 ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab2, 100])), 1639 ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab3, 100])), 1640 1641 ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab1, 100])), 1642 ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab2, 100])), 1643 ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab3, 100])), 1644 ?verify_mnesia(Nodes, []). 1645 1646%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1647 1648system_upgrade(doc) -> 1649 ["Test on-line and off-line upgrade of the Mnesia application"]. 1650 1651garb_decision(doc) -> 1652 ["Test that decisions are garbed correctly."]; 1653garb_decision(suite) -> []; 1654garb_decision(Config) when is_list(Config) -> 1655 [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), 1656 check_garb(Nodes), 1657 ?match({atomic, ok},mnesia:create_table(a, [{disc_copies, Nodes}])), 1658 check_garb(Nodes), 1659 ?match({atomic, ok},mnesia:create_table(b, [{ram_copies, Nodes}])), 1660 check_garb(Nodes), 1661 ?match({atomic, ok},mnesia:create_table(c, [{ram_copies, [Node1, Node3]}, 1662 {disc_copies, [Node2]}])), 1663 check_garb(Nodes), 1664 ?match({atomic, ok},mnesia:create_table(d, [{disc_copies, [Node1, Node3]}, 1665 {ram_copies, [Node2]}])), 1666 check_garb(Nodes), 1667 1668 W = fun(Tab) -> mnesia:write({Tab,1,1}) end, 1669 A = fun(Tab) -> mnesia:write({Tab,1,1}), exit(1) end, 1670 1671 ?match({atomic, ok}, mnesia:transaction(W,[a])), 1672 check_garb(Nodes), 1673 ?match({atomic, ok}, mnesia:transaction(W,[b])), 1674 check_garb(Nodes), 1675 ?match({atomic, ok}, mnesia:transaction(W,[c])), 1676 check_garb(Nodes), 1677 ?match({atomic, ok}, mnesia:transaction(W,[d])), 1678 check_garb(Nodes), 1679 ?match({aborted,1}, mnesia:transaction(A,[a])), 1680 check_garb(Nodes), 1681 ?match({aborted,1}, mnesia:transaction(A,[b])), 1682 check_garb(Nodes), 1683 ?match({aborted,1}, mnesia:transaction(A,[c])), 1684 check_garb(Nodes), 1685 ?match({aborted,1}, mnesia:transaction(A,[d])), 1686 check_garb(Nodes), 1687 1688 rpc:call(Node2, mnesia, lkill, []), 1689 ?match({atomic, ok}, mnesia:transaction(W,[a])), 1690 ?match({atomic, ok}, mnesia:transaction(W,[b])), 1691 ?match({atomic, ok}, mnesia:transaction(W,[c])), 1692 ?match({atomic, ok}, mnesia:transaction(W,[d])), 1693 check_garb(Nodes), 1694 ?match([], mnesia_test_lib:start_mnesia([Node2])), 1695 check_garb(Nodes), 1696 timer:sleep(2000), 1697 check_garb(Nodes), 1698 %%%%%% Check transient_decision logs %%%%% 1699 1700 ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1701 [{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)], 1702 ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1703 TD0 = mnesia_lib:val(latest_transient_decision), 1704 ?match(0, ets:info(TD0, size)), 1705 {atomic, ok} = mnesia:transaction(W,[a]), 1706 ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1707 ?match(TD0, mnesia_lib:val(latest_transient_decision)), 1708 [{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)], 1709 ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1710 ?match(false, TD0 =:= mnesia_lib:val(latest_transient_decision)), 1711 ?match(true, lists:member(TD0, mnesia_lib:val(previous_transient_decisions))), 1712 ?verify_mnesia(Nodes, []). 1713 1714check_garb(Nodes) -> 1715 rpc:multicall(Nodes, sys, get_status, [mnesia_recover]), 1716 ?match({_, []},rpc:multicall(Nodes, erlang, apply, [fun check_garb/0, []])). 1717 1718check_garb() -> 1719 try 1720 Ds = ets:tab2list(mnesia_decision), 1721 Check = fun({trans_tid,serial, _}) -> false; 1722 ({mnesia_down,_,_,_}) -> false; 1723 (_Else) -> true 1724 end, 1725 Node = node(), 1726 ?match({Node, []}, {node(), lists:filter(Check, Ds)}) 1727 catch _:_ -> ok 1728 end, 1729 ok. 1730