1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2008-2017. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%----------------------------------------------------------------
20%% Purpose:Test Suite for the 'pg2' module.
21%%-----------------------------------------------------------------
22-module(pg2_SUITE).
23
24-include_lib("common_test/include/ct.hrl").
25-define(datadir, proplists:get_value(data_dir, Config)).
26-define(privdir, proplists:get_value(priv_dir, Config)).
27
28-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
29	 init_per_group/2,end_per_group/2,
30	 init_per_testcase/2, end_per_testcase/2]).
31
32-export([
33	 otp_7277/1, otp_8259/1, otp_8653/1,
34         basic/1]).
35
36-define(TESTCASE, testcase_name).
37-define(testcase, proplists:get_value(?TESTCASE, Config)).
38
39%% Internal export.
40-export([mk_part_node_and_group/3, part2/4,
41         mk_part_node/3, part1/5, p_init/3, start_proc/1, sane/0]).
42
43init_per_testcase(Case, Config) ->
44    [{?TESTCASE, Case}| Config].
45
46end_per_testcase(_Case, _Config) ->
47    test_server_ctrl:kill_slavenodes(),
48    ok.
49
50suite() ->
51    [{ct_hooks,[ts_install_cth]},
52     {timetrap,{minutes,1}}].
53
54all() ->
55    [{group, tickets}].
56
57groups() ->
58    [{tickets, [],
59      [otp_7277, otp_8259, otp_8653, basic]}].
60
61init_per_suite(Config) ->
62    Config.
63
64end_per_suite(_Config) ->
65    ok.
66
67init_per_group(_GroupName, Config) ->
68    Config.
69
70end_per_group(_GroupName, Config) ->
71    Config.
72
73
74
75%% OTP-7277. Bugfix leave().
76otp_7277(Config) when is_list(Config) ->
77    ok = pg2:create(a),
78    ok = pg2:create(b),
79    P = spawn(forever()),
80    ok = pg2:join(a, P),
81    ok = pg2:leave(b, P),
82    true = exit(P, kill),
83    case {pg2:get_members(a), pg2:get_local_members(a)} of
84        {[], []} ->
85            ok;
86        _ ->
87            timer:sleep(100),
88            [] = pg2:get_members(a),
89            [] = pg2:get_local_members(a)
90    end,
91    _ = pg2:delete(a),
92    _ = pg2:delete(b),
93    ok.
94
95-define(UNTIL(Seq), loop_until_true(fun() -> Seq end, Config)).
96-define(UNTIL_LOOP, 300).
97
98%% OTP-8259. Member was not removed after being killed.
99otp_8653(Config) when is_list(Config) ->
100    [A, B, C] = start_nodes([a, b, c], peer, Config),
101
102    wait_for_ready_net(Config),
103
104    %% make b and c connected, partitioned from node() and a
105    rpc_cast(B, ?MODULE, part2, [Config, node(), A, C]),
106    ?UNTIL(is_ready_partition(Config)),
107
108    %% Connect to the other partition.
109    pong = net_adm:ping(B),
110    timer:sleep(100),
111    pong = net_adm:ping(C),
112    _ = global:sync(),
113    [A, B, C] = lists:sort(nodes()),
114
115    G = pg2_otp_8653,
116    ?UNTIL(begin
117	       GA = lists:sort(rpc:call(A, pg2, get_members, [G])),
118	       GB = lists:sort(rpc:call(B, pg2, get_members, [G])),
119	       GC = lists:sort(rpc:call(C, pg2, get_members, [G])),
120	       GT = lists:sort(pg2:get_members(G)),
121	       GA =:= GB andalso
122		   GB =:= GC andalso
123		   GC =:= GT andalso
124		   8 =:= length(GA)
125	   end),
126    ok = pg2:delete(G),
127    stop_nodes([A,B,C]),
128    ok.
129
130part2(Config, Main, A, C) ->
131    Function = mk_part_node_and_group,
132    case catch begin
133		   make_partition(Config, [Main, A], [node(), C], Function)
134	       end
135    of
136	ok -> ok
137    end.
138
139mk_part_node_and_group(File, MyPart0, Config) ->
140    touch(File, "start"), % debug
141    MyPart = lists:sort(MyPart0),
142    ?UNTIL(is_node_in_part(File, MyPart)),
143    G = pg2_otp_8653,
144    Pid = spawn(forever()),
145    ok = pg2:create(G),
146    _ = [ok = pg2:join(G, Pid) || _ <- [1,1]],
147    touch(File, "done").
148
149%% OTP-8259. Member was not removed after being killed.
150otp_8259(Config) when is_list(Config) ->
151    [A, B, C] = start_nodes([a, b, c], peer, Config),
152
153    wait_for_ready_net(Config),
154
155    G = pg2_otp_8259,
156    Name = otp_8259_a_global_name,
157
158    %% start different processes in both partitions
159    {Pid, yes} = rpc:call(A, ?MODULE, start_proc, [Name]),
160
161    ok = pg2:create(G),
162    ok = pg2:join(G, Pid),
163
164    %% make b and c connected, partitioned from node() and a
165    rpc_cast(B, ?MODULE, part1, [Config, node(), A, C, Name]),
166    ?UNTIL(is_ready_partition(Config)),
167
168    %% Connect to the other partition.
169    %% The resolver on node b will be called.
170    pong = net_adm:ping(B),
171    timer:sleep(100),
172    pong = net_adm:ping(C),
173    _ = global:sync(),
174    [A, B, C] = lists:sort(nodes()),
175
176    %% Pid has been killed by the resolver.
177    %% Pid has been removed from pg2 on all nodes, in particular node B.
178    ?UNTIL([] =:= rpc:call(B, pg2, get_members, [G])),
179    ?UNTIL([] =:= pg2:get_members(G)),
180    ?UNTIL([] =:= rpc:call(A, pg2, get_members, [G])),
181    ?UNTIL([] =:= rpc:call(C, pg2, get_members, [G])),
182
183    ok = pg2:delete(G),
184    stop_nodes([A,B,C]),
185    ok.
186
187part1(Config, Main, A, C, Name) ->
188    case catch begin
189		   make_partition(Config, [Main, A], [node(), C]),
190		   {_Pid, yes} = start_proc(Name)
191	       end of
192	{_, yes} -> ok
193    end.
194
195start_proc(Name) ->
196    Pid = spawn(?MODULE, p_init, [self(), Name, node()]),
197    receive
198	{Pid, Res} -> {Pid, Res}
199    end.
200
201p_init(Parent, Name, TestServer) ->
202    Resolve = fun(_Name, Pid1, Pid2) ->
203                      %% The pid on node a will be chosen.
204                      [{_,Min}, {_,Max}] =
205                          lists:sort([{node(Pid1),Pid1}, {node(Pid2),Pid2}]),
206                      %% b is connected to test_server.
207                      %% exit(Min, kill), % would ping a
208                      rpc:cast(TestServer, erlang, exit, [Min, kill]),
209		      Max
210              end,
211    X = global:register_name(Name, self(), Resolve),
212    Parent ! {self(),X},
213    loop().
214
215loop() ->
216    receive
217	die ->
218	    exit(normal)
219    end.
220
221%% OTP-8259. Some basic tests.
222basic(Config) when is_list(Config) ->
223    _ = [pg2:delete(G) || G <- pg2:which_groups()],
224    _ = [do(Cs, T, Config) || {T,Cs} <- ts()],
225    ok.
226
227ts() ->
228    [
229     {t1,
230      [{create,[a],ignore},
231       {which_groups,[],[a]},
232       {get_closest_pid,[a],{error, {no_process, a}}},
233       {delete,[a],ignore}]},
234     {t2,
235      [{create,[a],ignore},
236       {join,[a,self()],ok},
237       {get_closest_pid,[a],self()},
238       {delete,[a],ignore}]},
239     {t3,
240      [{create,[a],ignore},
241       {new,p1},
242       {leave,[a,p1],ok},
243       {join,[b,p1],{error,{no_such_group,b}}},
244       {leave,[b,p1],{error,{no_such_group,b}}},
245       {get_members,[c],{error,{no_such_group,c}}},
246       {get_local_members,[c],{error,{no_such_group,c}}},
247       {join,[a,p1],ok},
248       {leave,[a,p1],ok},
249       {join,[a,p1],ok},
250       {join,[a,p1],ok},
251       {create,[a],ignore},
252       {get_closest_pid,[a],p1},
253       {leave,[a,p1],ok},
254       {get_closest_pid,[a],p1},
255       {leave,[a,p1],ok},
256       {get_closest_pid,[a],{error,{no_process, a}}},
257       {kill,p1},
258       {delete,[a],ignore}]},
259     {t4,
260      [{create,[a],ignore},
261       {new,p1},
262       {join,[a,p1],ok},
263       {get_members,[a],[p1]},
264       {get_local_members,[a],[p1]},
265       {kill,p1},
266       {get_members,[a],[]},
267       {get_local_members,[a],[]},
268       {delete,[a],ignore}]},
269     {t5,
270      [{create,[a],ignore},
271       {nodeup,n1},
272       {create,[a],ignore},
273       {join,[a,self()],ok},
274       {new,n1,p1},
275       {n1,{create,[b],ignore}},
276       {join,[a,p1],ok},
277       {join,[b,p1],ok},
278       {n1,{which_groups,[],[a,b]}},
279       {n1,{join,[a,p1],ok}},
280       {n1,{join,[b,p1],ok}},
281       {leave,[a,self()],ok},
282       {n1,{leave,[a,self()],ok}}, % noop
283       {n1,{leave,[b,p1],ok}},
284       {leave,[b,p1],ok},
285       {kill,n1,p1},
286       {nodedown,n1},
287       {delete,[b],ignore},
288       {delete,[a],ignore}]},
289     {t6,
290      [{create,[a],ignore}, % otp_7277
291       {create,[b],ignore},
292       {new,p},
293       {join,[a,p],ok},
294       {leave,[b,p],ok},
295       {kill,p},
296       {get_members,[a],[]},
297       {get_local_members,[a],[]},
298       {delete,[a],ignore},
299       {delete,[b],ignore}]},
300     {t7, % p1 joins twice, the new node gets informed about that
301      [{create,[a],ignore},
302       {new,p1},
303       {join,[a,p1],ok},
304       {join,[a,p1],ok},
305       {get_members,[a],[p1,p1]},
306       {get_local_members,[a],[p1,p1]},
307       {nodeup,n1},
308       {leave,[a,p1],ok},
309       {get_members,[a],[p1]},
310       {get_local_members,[a],[p1]},
311       {n1,{get_members,[a],[p1]}},
312       {leave,[a,p1],ok},
313       {get_members,[a],[]},
314       {n1,{get_members,[a],[]}},
315       {nodedown,n1},
316       {delete,[a],ignore},
317       {kill,p1}]},
318     {t8,
319      [{create,[a],ignore},
320       {new,p1},
321       {join,[a,p1],ok},
322       {join,[a,p1],ok},
323       {delete,[a],ignore},
324       {get_members,[a],{error,{no_such_group,a}}},
325       {kill,p1}]}
326    ].
327
328do(Cs, T, Config) ->
329    io:format("*** Test ~p ***~n", [T]),
330    {ok,T} = (catch {do(Cs, [], [], Config),T}).
331
332do([{nodeup,N} | Cs], Ps, Ns, Config) ->
333    [TestNode] = start_nodes([N], peer, Config),
334    pr(node(), {nodeup,N,TestNode}),
335    global:sync(),
336    timer:sleep(100),
337    {ok,_} = rpc:call(TestNode, pg2, start, []),
338    NNs = [{N,TestNode} | Ns],
339    sane(NNs),
340    do(Cs, Ps, NNs, Config);
341do([{nodedown,N}=C | Cs], Ps, Ns, Config) ->
342    {N, TestNode} = lists:keyfind(N, 1, Ns),
343    stop_node(TestNode),
344    timer:sleep(100),
345    pr(node(), C),
346    do(Cs, Ps, lists:keydelete(N, 1, Ns), Config);
347do([{new,P} | Cs], Ps, Ns, Config) ->
348    NPs = new_proc(node(), P, Ps, Ns),
349    do(Cs, NPs, Ns, Config);
350do([{new,N,P} | Cs], Ps, Ns, Config) ->
351    NPs = new_proc(N, P, Ps, Ns),
352    do(Cs, NPs, Ns, Config);
353do([{kill,P} | Cs], Ps, Ns, Config) ->
354    NPs = killit(node(), P, Ps, Ns),
355    do(Cs, NPs, Ns, Config);
356do([{kill,N,P} | Cs], Ps, Ns, Config) ->
357    NPs = killit(N, P, Ps, Ns),
358    do(Cs, NPs, Ns, Config);
359do([{Node,{_,_,_}=C} | Cs], Ps, Ns, Config) ->
360    doit(Node, C, Ps, Ns),
361    do(Cs, Ps, Ns, Config);
362do([C | Cs], Ps, Ns, Config) ->
363    doit(node(), C, Ps, Ns),
364    do(Cs, Ps, Ns, Config);
365do([], Ps, Ns, _Config) ->
366    [] = Ns,
367    [] = Ps,
368    [] = pg2:which_groups(),
369    [] = ets:tab2list(pg2_table),
370    [] = nodes(),
371    ok.
372
373doit(N, C, Ps, Ns) ->
374    Node = get_node(N, Ns),
375    pr(Node, C),
376    {F,As,R} = replace_pids(C, Ps),
377    case rpc:call(Node, erlang, apply, [pg2, F, As]) of
378        Result when Result =:= R orelse R =:= ignore ->
379            sane(Ns);
380        Else ->
381            io:format("~p and ~p: expected ~p, but got ~p~n",
382                      [F, As, R, Else]),
383            throw({error,{F, As, R, Else}})
384    end.
385
386new_proc(N, P, Ps, Ns) ->
387    Node = get_node(N, Ns),
388    Pid = rpc:call(Node, erlang, spawn, [forever()]),
389    pr(Node, {new,P,Pid}),
390    [{P,Pid}|Ps].
391
392killit(N, P, Ps, Ns) ->
393    {P, Pid} = lists:keyfind(P, 1, Ps),
394    Node = get_node(N, Ns),
395    pr(Node, {kill,P,Pid}),
396    rpc:call(Node, erlang, exit, [Pid, kill]),
397    timer:sleep(100),
398    sane(Ns),
399    lists:keydelete(P, 1, Ps).
400
401pr(Node, C) ->
402    _ = [io:format("~p: ", [Node]) || Node =/= node()],
403    io:format("do ~p~n", [C]).
404
405get_node(N, Ns) ->
406    if
407        N =:= node() ->
408            node();
409        true ->
410            {N, TestNode} = lists:keyfind(N, 1, Ns),
411            TestNode
412    end.
413
414forever() ->
415    fun() -> receive after infinity -> ok end end.
416
417replace_pids(T, Ps) when is_tuple(T) ->
418    list_to_tuple(replace_pids(tuple_to_list(T), Ps));
419replace_pids([E | Es], Ps) ->
420    [replace_pids(E, Ps) | replace_pids(Es, Ps)];
421replace_pids(A, Ps) ->
422    case lists:keyfind(A, 1, Ps) of
423        {A, Pid} ->
424            Pid;
425        _ ->
426            A
427    end.
428
429sane(Ns) ->
430    Nodes = [node()] ++ [NN || {_,NN} <- Ns],
431    _ = [io:format("~p, pg2_table:~n   ~p~n",  % debug
432                   [N, rpc:call(N, ets, tab2list, [pg2_table])]) ||
433            N <- Nodes],
434    R = [case rpc:call(Node, ?MODULE, sane, []) of
435             {'EXIT',Error} ->
436                 {error, Node, Error};
437             _ ->
438                 ok
439         end || Node <- Nodes],
440    case lists:usort(R) of
441        [ok] -> wsane(Nodes);
442        _ -> throw(R)
443    end.
444
445wsane(Ns) ->
446    %% Same members on all nodes:
447    {[_],gs} =
448        {lists:usort([rpc:call(N, pg2, which_groups, []) || N <- Ns]),gs},
449    _ = [{[_],ms,G} = {lists:usort([rpc:call(N, pg2, get_members, [G]) ||
450                                       N <- Ns]),ms,G} ||
451            G <- pg2:which_groups()],
452    %% The local members are a partitioning of the members:
453    [begin
454         LocalMembers =
455             lists:sort(lists:append(
456                          [rpc:call(N, pg2, get_local_members, [G]) ||
457                              N <- Ns])),
458         {part, LocalMembers} = {part, lists:sort(pg2:get_members(G))}
459     end || G <- pg2:which_groups()],
460    %% The closest pid should run on the local node, if possible.
461    [[case rpc:call(N, pg2, get_closest_pid, [G]) of
462          Pid when is_pid(Pid), node(Pid) =:= N ->
463              true =
464                  lists:member(Pid, rpc:call(N, pg2, get_local_members, [G]));
465	  %% FIXME. Om annan nod: member, local = [].
466          _ -> [] = rpc:call(N, pg2, get_local_members, [G])
467      end || N <- Ns]
468     || G <- pg2:which_groups()].
469
470%% Look inside the pg2_table.
471sane() ->
472    L = ets:tab2list(pg2_table),
473    Gs = lists:sort([G || {{group,G}} <- L]),
474    MGs = lists:usort([G || {{member,G,_},_} <- L]),
475    MPs = lists:usort([P || {{member,_,P},_} <- L]),
476    {[],mg,MGs,Gs} = {MGs -- Gs,mg,MGs,Gs},
477    RPs = [P || {{ref,P},_RPid,_Ref,_C} <- L],
478    {MPs,rp} = {RPs,rp},
479    RPs2 = [P || {{ref,_Ref},P} <- L],
480    {MPs,rp2} = {RPs2,rp2},
481    _ = [true = C >= 1 || {{ref,_P},_RPid,_Ref,C} <- L],
482    LGs = lists:usort([G || {{local_member,G,_}} <- L]),
483    LPs = lists:usort([P || {{local_member,_,P}} <- L]),
484    {[],lg} = {LGs -- Gs,lg},
485    {[],lp} = {LPs -- MPs,lp},
486    PGs = lists:usort([G || {{pid,_,G}} <- L]),
487    PPs = lists:usort([P || {{pid,P,_}} <- L]),
488    {[],pg} = {PGs -- Gs,pg},
489    {MPs,pp} = {PPs,pp},
490    _ = [true = C >= 1 || {{member,_,_},C} <- L],
491    ok.
492
493%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
494
495%% Mostly copied from global_SUITE.erl
496%% (Setting up a partition is quite tricky.)
497
498loop_until_true(Fun, Config) ->
499    case Fun() of
500	true ->
501	    true;
502	_ ->
503            timer:sleep(?UNTIL_LOOP),
504            loop_until_true(Fun, Config)
505    end.
506
507start_node_rel(Name, Rel, How) ->
508    {Release, Compat} = case Rel of
509                            this ->
510                                {[this], "+R8"};
511                            Rel when is_atom(Rel) ->
512                                {[{release, atom_to_list(Rel)}], ""};
513                            RelList ->
514				{RelList, ""}
515			end,
516    Pa = filename:dirname(code:which(?MODULE)),
517    Res = test_server:start_node(Name, How,
518				 [{args,
519				   Compat ++
520				       " -kernel net_setuptime 100 "
521				   " -pa " ++ Pa},
522				  {erl, Release}]),
523    Res.
524
525start_nodes(L, How, Config) ->
526    start_nodes2(L, How, 0, Config),
527    Nodes = collect_nodes(0, length(L)),
528    ?UNTIL([] =:= Nodes -- nodes()),
529    %% Pinging doesn't help, we have to wait too, for nodes() to become
530    %% correct on the other node.
531    lists:foreach(fun(E) ->
532			  net_adm:ping(E)
533		  end,
534		  Nodes),
535    verify_nodes(Nodes, Config),
536    Nodes.
537
538verify_nodes(Nodes, Config) ->
539    verify_nodes(Nodes, lists:sort([node() | Nodes]), Config).
540
541verify_nodes([], _N, _Config) ->
542    [];
543verify_nodes([Node | Rest], N, Config) ->
544    ?UNTIL(
545       case rpc:call(Node, erlang, nodes, []) of
546	   Nodes when is_list(Nodes) ->
547	       case N =:= lists:sort([Node | Nodes]) of
548		   true ->
549		       true;
550		   false ->
551		       lists:foreach(fun(Nd) ->
552					     rpc:call(Nd, net_adm, ping,
553                                                      [Node])
554				     end,
555				     nodes()),
556		       false
557	       end;
558	   _ ->
559	       false
560       end
561      ),
562    verify_nodes(Rest, N, Config).
563
564
565start_nodes2([], _How, _, _Config) ->
566    [];
567start_nodes2([Name | Rest], How, N, Config) ->
568    Self = self(),
569    spawn(fun() ->
570		  erlang:display({starting, Name}),
571		  {ok, R} = start_node(Name, How, Config),
572		  erlang:display({started, Name, R}),
573		  Self ! {N, R},
574		  %% sleeping is necessary, or with peer nodes, they will
575		  %% go down again, despite {linked, false}.
576		  ct:sleep(100000)
577	  end),
578    start_nodes2(Rest, How, N+1, Config).
579
580collect_nodes(N, N) ->
581    [];
582collect_nodes(N, Max) ->
583    receive
584	{N, Node} ->
585            [Node | collect_nodes(N+1, Max)]
586    end.
587
588start_node(Name, How, Config) ->
589    start_node(Name, How, "", Config).
590
591start_node(Name0, How, Args, Config) ->
592    Name = node_name(Name0, Config),
593    Pa = filename:dirname(code:which(?MODULE)),
594    test_server:start_node(Name, How, [{args,
595                                        Args ++ " " ++
596					    "-kernel net_setuptime 100 "
597                                        "-noshell "
598                                        "-pa " ++ Pa},
599                                       {linked, false}]).
600stop_nodes(Nodes) ->
601    lists:foreach(fun(Node) -> stop_node(Node) end, Nodes).
602
603stop_node(Node) ->
604    test_server:stop_node(Node).
605
606get_known(Node) ->
607    case catch gen_server:call({global_name_server,Node},get_known,infinity) of
608        {'EXIT', _} ->
609            [list, without, nodenames];
610        Known when is_list(Known) ->
611            lists:sort([Node | Known])
612    end.
613
614node_name(Name, Config) ->
615    U = "_",
616    {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(now()),
617    Date = io_lib:format("~4w_~2..0w_~2..0w__~2..0w_~2..0w_~2..0w",
618                         [Y,M,D, H,Min,S]),
619    L = lists:flatten(Date),
620    lists:concat([Name,U,?testcase,U,U,L]).
621
622%% This one runs on one node in Part2.
623%% The partition is ready when is_ready_partition(Config) returns (true).
624make_partition(Config, Part1, Part2) ->
625    make_partition(Config, Part1, Part2, mk_part_node).
626
627make_partition(Config, Part1, Part2, Function) ->
628    Dir = proplists:get_value(priv_dir, Config),
629    Ns = [begin
630              Name = lists:concat([atom_to_list(N),"_",msec(),".part"]),
631              File = filename:join([Dir, Name]),
632              file:delete(File),
633              rpc_cast(N, ?MODULE, Function, [File, Part, Config], File),
634              {N, File}
635          end || Part <- [Part1, Part2], N <- Part],
636    all_nodes_files(Ns, "done", Config),
637    lists:foreach(fun({_N,File}) -> file:delete(File) end, Ns),
638    PartFile = make_partition_file(Config),
639    touch(PartFile, "done").
640
641%% The node signals its success by touching a file.
642mk_part_node(File, MyPart0, Config) ->
643    touch(File, "start"), % debug
644    MyPart = lists:sort(MyPart0),
645    ?UNTIL(is_node_in_part(File, MyPart)),
646    touch(File, "done").
647
648%% The calls to append_to_file are for debugging.
649is_node_in_part(File, MyPart) ->
650    lists:foreach(fun(N) ->
651                          _ = erlang:disconnect_node(N)
652                  end, nodes() -- MyPart),
653    case {(Known = get_known(node())) =:= MyPart,
654          (Nodes = lists:sort([node() | nodes()])) =:= MyPart} of
655        {true, true} ->
656            %% Make sure the resolvers have been terminated,
657            %% otherwise they may pop up and send some message.
658            %% (This check is probably unnecessary.)
659            case element(5, global:info()) of
660                [] ->
661                    true;
662                Rs ->
663                    append_to_file(File, {now(), Known, Nodes, Rs}),
664                    false
665            end;
666        _ ->
667            append_to_file(File, {now(), Known, Nodes}),
668            false
669    end.
670
671is_ready_partition(Config) ->
672    File = make_partition_file(Config),
673    file_contents(File, "done", Config),
674    file:delete(File),
675    true.
676
677wait_for_ready_net(Config) ->
678    wait_for_ready_net([node()|nodes()], Config).
679
680wait_for_ready_net(Nodes0, Config) ->
681    Nodes = lists:sort(Nodes0),
682    io:format("wait_for_ready_net ~p~n", [Nodes]),
683    ?UNTIL(begin
684               lists:all(fun(N) -> Nodes =:= get_known(N) end, Nodes) and
685		   lists:all(fun(N) ->
686				     LNs = rpc:call(N, erlang, nodes, []),
687				     Nodes =:= lists:sort([N | LNs])
688			     end, Nodes)
689           end).
690
691%% To make it less probable that some low-level problem causes
692%% problems, the receiving node is ping:ed.
693rpc_cast(Node, Module, Function, Args) ->
694    {_,pong,Node}= {node(),net_adm:ping(Node),Node},
695    rpc:cast(Node, Module, Function, Args).
696
697rpc_cast(Node, Module, Function, Args, File) ->
698    case net_adm:ping(Node) of
699        pong ->
700            rpc:cast(Node, Module, Function, Args);
701        Else ->
702            append_to_file(File, {now(), {rpc_cast, Node, Module, Function,
703                                          Args, Else}})
704            %% Maybe we should crash, but it probably doesn't matter.
705    end.
706
707touch(File, List) ->
708    ok = file:write_file(File, list_to_binary(List)).
709
710append_to_file(File, Term) ->
711    {ok, Fd} = file:open(File, [raw,binary,append]),
712    ok = file:write(Fd, io_lib:format("~p.~n", [Term])),
713    ok = file:close(Fd).
714
715all_nodes_files(Files, ContentsList, Config) ->
716    lists:all(fun({_N,File}) ->
717                      file_contents(File, ContentsList, Config)
718              end, Files).
719
720file_contents(File, ContentsList, Config) ->
721    file_contents(File, ContentsList, Config, no_log_file).
722
723file_contents(File, ContentsList, Config, LogFile) ->
724    Contents = list_to_binary(ContentsList),
725    Sz = size(Contents),
726    ?UNTIL(begin
727               case file:read_file(File) of
728                   {ok, FileContents}=Reply ->
729                       case catch split_binary(FileContents, Sz) of
730                           {Contents,_} ->
731                               true;
732                           _ ->
733                               catch append_to_file(LogFile,
734                                                    {File,Contents,Reply}),
735                               false
736                       end;
737                   Reply ->
738                       catch append_to_file(LogFile, {File, Contents, Reply}),
739                       false
740               end
741           end).
742
743make_partition_file(Config) ->
744    Dir = proplists:get_value(priv_dir, Config),
745    filename:join([Dir, atom_to_list(make_partition_done)]).
746
747msec() ->
748    msec(now()).
749
750msec(T) ->
751    element(1,T)*1000000000 + element(2,T)*1000 + element(3,T) div 1000.
752