1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2008-2017. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%%---------------------------------------------------------------- 20%% Purpose:Test Suite for the 'pg2' module. 21%%----------------------------------------------------------------- 22-module(pg2_SUITE). 23 24-include_lib("common_test/include/ct.hrl"). 25-define(datadir, proplists:get_value(data_dir, Config)). 26-define(privdir, proplists:get_value(priv_dir, Config)). 27 28-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 29 init_per_group/2,end_per_group/2, 30 init_per_testcase/2, end_per_testcase/2]). 31 32-export([ 33 otp_7277/1, otp_8259/1, otp_8653/1, 34 basic/1]). 35 36-define(TESTCASE, testcase_name). 37-define(testcase, proplists:get_value(?TESTCASE, Config)). 38 39%% Internal export. 40-export([mk_part_node_and_group/3, part2/4, 41 mk_part_node/3, part1/5, p_init/3, start_proc/1, sane/0]). 42 43init_per_testcase(Case, Config) -> 44 [{?TESTCASE, Case}| Config]. 45 46end_per_testcase(_Case, _Config) -> 47 test_server_ctrl:kill_slavenodes(), 48 ok. 49 50suite() -> 51 [{ct_hooks,[ts_install_cth]}, 52 {timetrap,{minutes,1}}]. 53 54all() -> 55 [{group, tickets}]. 56 57groups() -> 58 [{tickets, [], 59 [otp_7277, otp_8259, otp_8653, basic]}]. 60 61init_per_suite(Config) -> 62 Config. 63 64end_per_suite(_Config) -> 65 ok. 66 67init_per_group(_GroupName, Config) -> 68 Config. 69 70end_per_group(_GroupName, Config) -> 71 Config. 72 73 74 75%% OTP-7277. Bugfix leave(). 76otp_7277(Config) when is_list(Config) -> 77 ok = pg2:create(a), 78 ok = pg2:create(b), 79 P = spawn(forever()), 80 ok = pg2:join(a, P), 81 ok = pg2:leave(b, P), 82 true = exit(P, kill), 83 case {pg2:get_members(a), pg2:get_local_members(a)} of 84 {[], []} -> 85 ok; 86 _ -> 87 timer:sleep(100), 88 [] = pg2:get_members(a), 89 [] = pg2:get_local_members(a) 90 end, 91 _ = pg2:delete(a), 92 _ = pg2:delete(b), 93 ok. 94 95-define(UNTIL(Seq), loop_until_true(fun() -> Seq end, Config)). 96-define(UNTIL_LOOP, 300). 97 98%% OTP-8259. Member was not removed after being killed. 99otp_8653(Config) when is_list(Config) -> 100 [A, B, C] = start_nodes([a, b, c], peer, Config), 101 102 wait_for_ready_net(Config), 103 104 %% make b and c connected, partitioned from node() and a 105 rpc_cast(B, ?MODULE, part2, [Config, node(), A, C]), 106 ?UNTIL(is_ready_partition(Config)), 107 108 %% Connect to the other partition. 109 pong = net_adm:ping(B), 110 timer:sleep(100), 111 pong = net_adm:ping(C), 112 _ = global:sync(), 113 [A, B, C] = lists:sort(nodes()), 114 115 G = pg2_otp_8653, 116 ?UNTIL(begin 117 GA = lists:sort(rpc:call(A, pg2, get_members, [G])), 118 GB = lists:sort(rpc:call(B, pg2, get_members, [G])), 119 GC = lists:sort(rpc:call(C, pg2, get_members, [G])), 120 GT = lists:sort(pg2:get_members(G)), 121 GA =:= GB andalso 122 GB =:= GC andalso 123 GC =:= GT andalso 124 8 =:= length(GA) 125 end), 126 ok = pg2:delete(G), 127 stop_nodes([A,B,C]), 128 ok. 129 130part2(Config, Main, A, C) -> 131 Function = mk_part_node_and_group, 132 case catch begin 133 make_partition(Config, [Main, A], [node(), C], Function) 134 end 135 of 136 ok -> ok 137 end. 138 139mk_part_node_and_group(File, MyPart0, Config) -> 140 touch(File, "start"), % debug 141 MyPart = lists:sort(MyPart0), 142 ?UNTIL(is_node_in_part(File, MyPart)), 143 G = pg2_otp_8653, 144 Pid = spawn(forever()), 145 ok = pg2:create(G), 146 _ = [ok = pg2:join(G, Pid) || _ <- [1,1]], 147 touch(File, "done"). 148 149%% OTP-8259. Member was not removed after being killed. 150otp_8259(Config) when is_list(Config) -> 151 [A, B, C] = start_nodes([a, b, c], peer, Config), 152 153 wait_for_ready_net(Config), 154 155 G = pg2_otp_8259, 156 Name = otp_8259_a_global_name, 157 158 %% start different processes in both partitions 159 {Pid, yes} = rpc:call(A, ?MODULE, start_proc, [Name]), 160 161 ok = pg2:create(G), 162 ok = pg2:join(G, Pid), 163 164 %% make b and c connected, partitioned from node() and a 165 rpc_cast(B, ?MODULE, part1, [Config, node(), A, C, Name]), 166 ?UNTIL(is_ready_partition(Config)), 167 168 %% Connect to the other partition. 169 %% The resolver on node b will be called. 170 pong = net_adm:ping(B), 171 timer:sleep(100), 172 pong = net_adm:ping(C), 173 _ = global:sync(), 174 [A, B, C] = lists:sort(nodes()), 175 176 %% Pid has been killed by the resolver. 177 %% Pid has been removed from pg2 on all nodes, in particular node B. 178 ?UNTIL([] =:= rpc:call(B, pg2, get_members, [G])), 179 ?UNTIL([] =:= pg2:get_members(G)), 180 ?UNTIL([] =:= rpc:call(A, pg2, get_members, [G])), 181 ?UNTIL([] =:= rpc:call(C, pg2, get_members, [G])), 182 183 ok = pg2:delete(G), 184 stop_nodes([A,B,C]), 185 ok. 186 187part1(Config, Main, A, C, Name) -> 188 case catch begin 189 make_partition(Config, [Main, A], [node(), C]), 190 {_Pid, yes} = start_proc(Name) 191 end of 192 {_, yes} -> ok 193 end. 194 195start_proc(Name) -> 196 Pid = spawn(?MODULE, p_init, [self(), Name, node()]), 197 receive 198 {Pid, Res} -> {Pid, Res} 199 end. 200 201p_init(Parent, Name, TestServer) -> 202 Resolve = fun(_Name, Pid1, Pid2) -> 203 %% The pid on node a will be chosen. 204 [{_,Min}, {_,Max}] = 205 lists:sort([{node(Pid1),Pid1}, {node(Pid2),Pid2}]), 206 %% b is connected to test_server. 207 %% exit(Min, kill), % would ping a 208 rpc:cast(TestServer, erlang, exit, [Min, kill]), 209 Max 210 end, 211 X = global:register_name(Name, self(), Resolve), 212 Parent ! {self(),X}, 213 loop(). 214 215loop() -> 216 receive 217 die -> 218 exit(normal) 219 end. 220 221%% OTP-8259. Some basic tests. 222basic(Config) when is_list(Config) -> 223 _ = [pg2:delete(G) || G <- pg2:which_groups()], 224 _ = [do(Cs, T, Config) || {T,Cs} <- ts()], 225 ok. 226 227ts() -> 228 [ 229 {t1, 230 [{create,[a],ignore}, 231 {which_groups,[],[a]}, 232 {get_closest_pid,[a],{error, {no_process, a}}}, 233 {delete,[a],ignore}]}, 234 {t2, 235 [{create,[a],ignore}, 236 {join,[a,self()],ok}, 237 {get_closest_pid,[a],self()}, 238 {delete,[a],ignore}]}, 239 {t3, 240 [{create,[a],ignore}, 241 {new,p1}, 242 {leave,[a,p1],ok}, 243 {join,[b,p1],{error,{no_such_group,b}}}, 244 {leave,[b,p1],{error,{no_such_group,b}}}, 245 {get_members,[c],{error,{no_such_group,c}}}, 246 {get_local_members,[c],{error,{no_such_group,c}}}, 247 {join,[a,p1],ok}, 248 {leave,[a,p1],ok}, 249 {join,[a,p1],ok}, 250 {join,[a,p1],ok}, 251 {create,[a],ignore}, 252 {get_closest_pid,[a],p1}, 253 {leave,[a,p1],ok}, 254 {get_closest_pid,[a],p1}, 255 {leave,[a,p1],ok}, 256 {get_closest_pid,[a],{error,{no_process, a}}}, 257 {kill,p1}, 258 {delete,[a],ignore}]}, 259 {t4, 260 [{create,[a],ignore}, 261 {new,p1}, 262 {join,[a,p1],ok}, 263 {get_members,[a],[p1]}, 264 {get_local_members,[a],[p1]}, 265 {kill,p1}, 266 {get_members,[a],[]}, 267 {get_local_members,[a],[]}, 268 {delete,[a],ignore}]}, 269 {t5, 270 [{create,[a],ignore}, 271 {nodeup,n1}, 272 {create,[a],ignore}, 273 {join,[a,self()],ok}, 274 {new,n1,p1}, 275 {n1,{create,[b],ignore}}, 276 {join,[a,p1],ok}, 277 {join,[b,p1],ok}, 278 {n1,{which_groups,[],[a,b]}}, 279 {n1,{join,[a,p1],ok}}, 280 {n1,{join,[b,p1],ok}}, 281 {leave,[a,self()],ok}, 282 {n1,{leave,[a,self()],ok}}, % noop 283 {n1,{leave,[b,p1],ok}}, 284 {leave,[b,p1],ok}, 285 {kill,n1,p1}, 286 {nodedown,n1}, 287 {delete,[b],ignore}, 288 {delete,[a],ignore}]}, 289 {t6, 290 [{create,[a],ignore}, % otp_7277 291 {create,[b],ignore}, 292 {new,p}, 293 {join,[a,p],ok}, 294 {leave,[b,p],ok}, 295 {kill,p}, 296 {get_members,[a],[]}, 297 {get_local_members,[a],[]}, 298 {delete,[a],ignore}, 299 {delete,[b],ignore}]}, 300 {t7, % p1 joins twice, the new node gets informed about that 301 [{create,[a],ignore}, 302 {new,p1}, 303 {join,[a,p1],ok}, 304 {join,[a,p1],ok}, 305 {get_members,[a],[p1,p1]}, 306 {get_local_members,[a],[p1,p1]}, 307 {nodeup,n1}, 308 {leave,[a,p1],ok}, 309 {get_members,[a],[p1]}, 310 {get_local_members,[a],[p1]}, 311 {n1,{get_members,[a],[p1]}}, 312 {leave,[a,p1],ok}, 313 {get_members,[a],[]}, 314 {n1,{get_members,[a],[]}}, 315 {nodedown,n1}, 316 {delete,[a],ignore}, 317 {kill,p1}]}, 318 {t8, 319 [{create,[a],ignore}, 320 {new,p1}, 321 {join,[a,p1],ok}, 322 {join,[a,p1],ok}, 323 {delete,[a],ignore}, 324 {get_members,[a],{error,{no_such_group,a}}}, 325 {kill,p1}]} 326 ]. 327 328do(Cs, T, Config) -> 329 io:format("*** Test ~p ***~n", [T]), 330 {ok,T} = (catch {do(Cs, [], [], Config),T}). 331 332do([{nodeup,N} | Cs], Ps, Ns, Config) -> 333 [TestNode] = start_nodes([N], peer, Config), 334 pr(node(), {nodeup,N,TestNode}), 335 global:sync(), 336 timer:sleep(100), 337 {ok,_} = rpc:call(TestNode, pg2, start, []), 338 NNs = [{N,TestNode} | Ns], 339 sane(NNs), 340 do(Cs, Ps, NNs, Config); 341do([{nodedown,N}=C | Cs], Ps, Ns, Config) -> 342 {N, TestNode} = lists:keyfind(N, 1, Ns), 343 stop_node(TestNode), 344 timer:sleep(100), 345 pr(node(), C), 346 do(Cs, Ps, lists:keydelete(N, 1, Ns), Config); 347do([{new,P} | Cs], Ps, Ns, Config) -> 348 NPs = new_proc(node(), P, Ps, Ns), 349 do(Cs, NPs, Ns, Config); 350do([{new,N,P} | Cs], Ps, Ns, Config) -> 351 NPs = new_proc(N, P, Ps, Ns), 352 do(Cs, NPs, Ns, Config); 353do([{kill,P} | Cs], Ps, Ns, Config) -> 354 NPs = killit(node(), P, Ps, Ns), 355 do(Cs, NPs, Ns, Config); 356do([{kill,N,P} | Cs], Ps, Ns, Config) -> 357 NPs = killit(N, P, Ps, Ns), 358 do(Cs, NPs, Ns, Config); 359do([{Node,{_,_,_}=C} | Cs], Ps, Ns, Config) -> 360 doit(Node, C, Ps, Ns), 361 do(Cs, Ps, Ns, Config); 362do([C | Cs], Ps, Ns, Config) -> 363 doit(node(), C, Ps, Ns), 364 do(Cs, Ps, Ns, Config); 365do([], Ps, Ns, _Config) -> 366 [] = Ns, 367 [] = Ps, 368 [] = pg2:which_groups(), 369 [] = ets:tab2list(pg2_table), 370 [] = nodes(), 371 ok. 372 373doit(N, C, Ps, Ns) -> 374 Node = get_node(N, Ns), 375 pr(Node, C), 376 {F,As,R} = replace_pids(C, Ps), 377 case rpc:call(Node, erlang, apply, [pg2, F, As]) of 378 Result when Result =:= R orelse R =:= ignore -> 379 sane(Ns); 380 Else -> 381 io:format("~p and ~p: expected ~p, but got ~p~n", 382 [F, As, R, Else]), 383 throw({error,{F, As, R, Else}}) 384 end. 385 386new_proc(N, P, Ps, Ns) -> 387 Node = get_node(N, Ns), 388 Pid = rpc:call(Node, erlang, spawn, [forever()]), 389 pr(Node, {new,P,Pid}), 390 [{P,Pid}|Ps]. 391 392killit(N, P, Ps, Ns) -> 393 {P, Pid} = lists:keyfind(P, 1, Ps), 394 Node = get_node(N, Ns), 395 pr(Node, {kill,P,Pid}), 396 rpc:call(Node, erlang, exit, [Pid, kill]), 397 timer:sleep(100), 398 sane(Ns), 399 lists:keydelete(P, 1, Ps). 400 401pr(Node, C) -> 402 _ = [io:format("~p: ", [Node]) || Node =/= node()], 403 io:format("do ~p~n", [C]). 404 405get_node(N, Ns) -> 406 if 407 N =:= node() -> 408 node(); 409 true -> 410 {N, TestNode} = lists:keyfind(N, 1, Ns), 411 TestNode 412 end. 413 414forever() -> 415 fun() -> receive after infinity -> ok end end. 416 417replace_pids(T, Ps) when is_tuple(T) -> 418 list_to_tuple(replace_pids(tuple_to_list(T), Ps)); 419replace_pids([E | Es], Ps) -> 420 [replace_pids(E, Ps) | replace_pids(Es, Ps)]; 421replace_pids(A, Ps) -> 422 case lists:keyfind(A, 1, Ps) of 423 {A, Pid} -> 424 Pid; 425 _ -> 426 A 427 end. 428 429sane(Ns) -> 430 Nodes = [node()] ++ [NN || {_,NN} <- Ns], 431 _ = [io:format("~p, pg2_table:~n ~p~n", % debug 432 [N, rpc:call(N, ets, tab2list, [pg2_table])]) || 433 N <- Nodes], 434 R = [case rpc:call(Node, ?MODULE, sane, []) of 435 {'EXIT',Error} -> 436 {error, Node, Error}; 437 _ -> 438 ok 439 end || Node <- Nodes], 440 case lists:usort(R) of 441 [ok] -> wsane(Nodes); 442 _ -> throw(R) 443 end. 444 445wsane(Ns) -> 446 %% Same members on all nodes: 447 {[_],gs} = 448 {lists:usort([rpc:call(N, pg2, which_groups, []) || N <- Ns]),gs}, 449 _ = [{[_],ms,G} = {lists:usort([rpc:call(N, pg2, get_members, [G]) || 450 N <- Ns]),ms,G} || 451 G <- pg2:which_groups()], 452 %% The local members are a partitioning of the members: 453 [begin 454 LocalMembers = 455 lists:sort(lists:append( 456 [rpc:call(N, pg2, get_local_members, [G]) || 457 N <- Ns])), 458 {part, LocalMembers} = {part, lists:sort(pg2:get_members(G))} 459 end || G <- pg2:which_groups()], 460 %% The closest pid should run on the local node, if possible. 461 [[case rpc:call(N, pg2, get_closest_pid, [G]) of 462 Pid when is_pid(Pid), node(Pid) =:= N -> 463 true = 464 lists:member(Pid, rpc:call(N, pg2, get_local_members, [G])); 465 %% FIXME. Om annan nod: member, local = []. 466 _ -> [] = rpc:call(N, pg2, get_local_members, [G]) 467 end || N <- Ns] 468 || G <- pg2:which_groups()]. 469 470%% Look inside the pg2_table. 471sane() -> 472 L = ets:tab2list(pg2_table), 473 Gs = lists:sort([G || {{group,G}} <- L]), 474 MGs = lists:usort([G || {{member,G,_},_} <- L]), 475 MPs = lists:usort([P || {{member,_,P},_} <- L]), 476 {[],mg,MGs,Gs} = {MGs -- Gs,mg,MGs,Gs}, 477 RPs = [P || {{ref,P},_RPid,_Ref,_C} <- L], 478 {MPs,rp} = {RPs,rp}, 479 RPs2 = [P || {{ref,_Ref},P} <- L], 480 {MPs,rp2} = {RPs2,rp2}, 481 _ = [true = C >= 1 || {{ref,_P},_RPid,_Ref,C} <- L], 482 LGs = lists:usort([G || {{local_member,G,_}} <- L]), 483 LPs = lists:usort([P || {{local_member,_,P}} <- L]), 484 {[],lg} = {LGs -- Gs,lg}, 485 {[],lp} = {LPs -- MPs,lp}, 486 PGs = lists:usort([G || {{pid,_,G}} <- L]), 487 PPs = lists:usort([P || {{pid,P,_}} <- L]), 488 {[],pg} = {PGs -- Gs,pg}, 489 {MPs,pp} = {PPs,pp}, 490 _ = [true = C >= 1 || {{member,_,_},C} <- L], 491 ok. 492 493%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 494 495%% Mostly copied from global_SUITE.erl 496%% (Setting up a partition is quite tricky.) 497 498loop_until_true(Fun, Config) -> 499 case Fun() of 500 true -> 501 true; 502 _ -> 503 timer:sleep(?UNTIL_LOOP), 504 loop_until_true(Fun, Config) 505 end. 506 507start_node_rel(Name, Rel, How) -> 508 {Release, Compat} = case Rel of 509 this -> 510 {[this], "+R8"}; 511 Rel when is_atom(Rel) -> 512 {[{release, atom_to_list(Rel)}], ""}; 513 RelList -> 514 {RelList, ""} 515 end, 516 Pa = filename:dirname(code:which(?MODULE)), 517 Res = test_server:start_node(Name, How, 518 [{args, 519 Compat ++ 520 " -kernel net_setuptime 100 " 521 " -pa " ++ Pa}, 522 {erl, Release}]), 523 Res. 524 525start_nodes(L, How, Config) -> 526 start_nodes2(L, How, 0, Config), 527 Nodes = collect_nodes(0, length(L)), 528 ?UNTIL([] =:= Nodes -- nodes()), 529 %% Pinging doesn't help, we have to wait too, for nodes() to become 530 %% correct on the other node. 531 lists:foreach(fun(E) -> 532 net_adm:ping(E) 533 end, 534 Nodes), 535 verify_nodes(Nodes, Config), 536 Nodes. 537 538verify_nodes(Nodes, Config) -> 539 verify_nodes(Nodes, lists:sort([node() | Nodes]), Config). 540 541verify_nodes([], _N, _Config) -> 542 []; 543verify_nodes([Node | Rest], N, Config) -> 544 ?UNTIL( 545 case rpc:call(Node, erlang, nodes, []) of 546 Nodes when is_list(Nodes) -> 547 case N =:= lists:sort([Node | Nodes]) of 548 true -> 549 true; 550 false -> 551 lists:foreach(fun(Nd) -> 552 rpc:call(Nd, net_adm, ping, 553 [Node]) 554 end, 555 nodes()), 556 false 557 end; 558 _ -> 559 false 560 end 561 ), 562 verify_nodes(Rest, N, Config). 563 564 565start_nodes2([], _How, _, _Config) -> 566 []; 567start_nodes2([Name | Rest], How, N, Config) -> 568 Self = self(), 569 spawn(fun() -> 570 erlang:display({starting, Name}), 571 {ok, R} = start_node(Name, How, Config), 572 erlang:display({started, Name, R}), 573 Self ! {N, R}, 574 %% sleeping is necessary, or with peer nodes, they will 575 %% go down again, despite {linked, false}. 576 ct:sleep(100000) 577 end), 578 start_nodes2(Rest, How, N+1, Config). 579 580collect_nodes(N, N) -> 581 []; 582collect_nodes(N, Max) -> 583 receive 584 {N, Node} -> 585 [Node | collect_nodes(N+1, Max)] 586 end. 587 588start_node(Name, How, Config) -> 589 start_node(Name, How, "", Config). 590 591start_node(Name0, How, Args, Config) -> 592 Name = node_name(Name0, Config), 593 Pa = filename:dirname(code:which(?MODULE)), 594 test_server:start_node(Name, How, [{args, 595 Args ++ " " ++ 596 "-kernel net_setuptime 100 " 597 "-noshell " 598 "-pa " ++ Pa}, 599 {linked, false}]). 600stop_nodes(Nodes) -> 601 lists:foreach(fun(Node) -> stop_node(Node) end, Nodes). 602 603stop_node(Node) -> 604 test_server:stop_node(Node). 605 606get_known(Node) -> 607 case catch gen_server:call({global_name_server,Node},get_known,infinity) of 608 {'EXIT', _} -> 609 [list, without, nodenames]; 610 Known when is_list(Known) -> 611 lists:sort([Node | Known]) 612 end. 613 614node_name(Name, Config) -> 615 U = "_", 616 {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(now()), 617 Date = io_lib:format("~4w_~2..0w_~2..0w__~2..0w_~2..0w_~2..0w", 618 [Y,M,D, H,Min,S]), 619 L = lists:flatten(Date), 620 lists:concat([Name,U,?testcase,U,U,L]). 621 622%% This one runs on one node in Part2. 623%% The partition is ready when is_ready_partition(Config) returns (true). 624make_partition(Config, Part1, Part2) -> 625 make_partition(Config, Part1, Part2, mk_part_node). 626 627make_partition(Config, Part1, Part2, Function) -> 628 Dir = proplists:get_value(priv_dir, Config), 629 Ns = [begin 630 Name = lists:concat([atom_to_list(N),"_",msec(),".part"]), 631 File = filename:join([Dir, Name]), 632 file:delete(File), 633 rpc_cast(N, ?MODULE, Function, [File, Part, Config], File), 634 {N, File} 635 end || Part <- [Part1, Part2], N <- Part], 636 all_nodes_files(Ns, "done", Config), 637 lists:foreach(fun({_N,File}) -> file:delete(File) end, Ns), 638 PartFile = make_partition_file(Config), 639 touch(PartFile, "done"). 640 641%% The node signals its success by touching a file. 642mk_part_node(File, MyPart0, Config) -> 643 touch(File, "start"), % debug 644 MyPart = lists:sort(MyPart0), 645 ?UNTIL(is_node_in_part(File, MyPart)), 646 touch(File, "done"). 647 648%% The calls to append_to_file are for debugging. 649is_node_in_part(File, MyPart) -> 650 lists:foreach(fun(N) -> 651 _ = erlang:disconnect_node(N) 652 end, nodes() -- MyPart), 653 case {(Known = get_known(node())) =:= MyPart, 654 (Nodes = lists:sort([node() | nodes()])) =:= MyPart} of 655 {true, true} -> 656 %% Make sure the resolvers have been terminated, 657 %% otherwise they may pop up and send some message. 658 %% (This check is probably unnecessary.) 659 case element(5, global:info()) of 660 [] -> 661 true; 662 Rs -> 663 append_to_file(File, {now(), Known, Nodes, Rs}), 664 false 665 end; 666 _ -> 667 append_to_file(File, {now(), Known, Nodes}), 668 false 669 end. 670 671is_ready_partition(Config) -> 672 File = make_partition_file(Config), 673 file_contents(File, "done", Config), 674 file:delete(File), 675 true. 676 677wait_for_ready_net(Config) -> 678 wait_for_ready_net([node()|nodes()], Config). 679 680wait_for_ready_net(Nodes0, Config) -> 681 Nodes = lists:sort(Nodes0), 682 io:format("wait_for_ready_net ~p~n", [Nodes]), 683 ?UNTIL(begin 684 lists:all(fun(N) -> Nodes =:= get_known(N) end, Nodes) and 685 lists:all(fun(N) -> 686 LNs = rpc:call(N, erlang, nodes, []), 687 Nodes =:= lists:sort([N | LNs]) 688 end, Nodes) 689 end). 690 691%% To make it less probable that some low-level problem causes 692%% problems, the receiving node is ping:ed. 693rpc_cast(Node, Module, Function, Args) -> 694 {_,pong,Node}= {node(),net_adm:ping(Node),Node}, 695 rpc:cast(Node, Module, Function, Args). 696 697rpc_cast(Node, Module, Function, Args, File) -> 698 case net_adm:ping(Node) of 699 pong -> 700 rpc:cast(Node, Module, Function, Args); 701 Else -> 702 append_to_file(File, {now(), {rpc_cast, Node, Module, Function, 703 Args, Else}}) 704 %% Maybe we should crash, but it probably doesn't matter. 705 end. 706 707touch(File, List) -> 708 ok = file:write_file(File, list_to_binary(List)). 709 710append_to_file(File, Term) -> 711 {ok, Fd} = file:open(File, [raw,binary,append]), 712 ok = file:write(Fd, io_lib:format("~p.~n", [Term])), 713 ok = file:close(Fd). 714 715all_nodes_files(Files, ContentsList, Config) -> 716 lists:all(fun({_N,File}) -> 717 file_contents(File, ContentsList, Config) 718 end, Files). 719 720file_contents(File, ContentsList, Config) -> 721 file_contents(File, ContentsList, Config, no_log_file). 722 723file_contents(File, ContentsList, Config, LogFile) -> 724 Contents = list_to_binary(ContentsList), 725 Sz = size(Contents), 726 ?UNTIL(begin 727 case file:read_file(File) of 728 {ok, FileContents}=Reply -> 729 case catch split_binary(FileContents, Sz) of 730 {Contents,_} -> 731 true; 732 _ -> 733 catch append_to_file(LogFile, 734 {File,Contents,Reply}), 735 false 736 end; 737 Reply -> 738 catch append_to_file(LogFile, {File, Contents, Reply}), 739 false 740 end 741 end). 742 743make_partition_file(Config) -> 744 Dir = proplists:get_value(priv_dir, Config), 745 filename:join([Dir, atom_to_list(make_partition_done)]). 746 747msec() -> 748 msec(now()). 749 750msec(T) -> 751 element(1,T)*1000000000 + element(2,T)*1000 + element(3,T) div 1000. 752