1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2020. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20-module(erpc_SUITE).
21
22-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
23	 init_per_group/2,end_per_group/2]).
24-export([call/1, call_reqtmo/1, call_against_old_node/1, cast/1,
25         send_request/1, send_request_fun/1,
26         send_request_receive_reqtmo/1,
27         send_request_wait_reqtmo/1,
28         send_request_check_reqtmo/1,
29         send_request_against_old_node/1,
30         multicall/1, multicall_reqtmo/1,
31         multicall_recv_opt/1,
32         multicall_recv_opt2/1,
33         multicall_recv_opt3/1,
34         multicast/1,
35         timeout_limit/1]).
36-export([init_per_testcase/2, end_per_testcase/2]).
37
38-export([call_func1/1, call_func2/1, call_func4/4]).
39
40-export([f/0, f2/0]).
41
42-include_lib("common_test/include/ct.hrl").
43
44suite() ->
45    [{ct_hooks,[ts_install_cth]},
46     {timetrap,{minutes,2}}].
47
48all() ->
49    [call,
50     call_reqtmo,
51     call_against_old_node,
52     cast,
53     send_request,
54     send_request_fun,
55     send_request_receive_reqtmo,
56     send_request_wait_reqtmo,
57     send_request_check_reqtmo,
58     send_request_against_old_node,
59     multicall,
60     multicall_reqtmo,
61     multicall_recv_opt,
62     multicall_recv_opt2,
63     multicall_recv_opt3,
64     multicast,
65     timeout_limit].
66
67groups() ->
68    [].
69
70init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
71    [{testcase, Func}|Config].
72
73end_per_testcase(_Func, _Config) ->
74    ok.
75
76init_per_suite(Config) ->
77    Config.
78
79end_per_suite(_Config) ->
80    ok.
81
82init_per_group(_GroupName, Config) ->
83    Config.
84
85end_per_group(_GroupName, Config) ->
86    Config.
87
88call(Config) when is_list(Config) ->
89    call_test(Config).
90
91call_test(Config) ->
92    call_test(node(), 10000),
93    call_test(node(), infinity),
94    try
95        erpc:call(node(), timer, sleep, [100], 10),
96        ct:fail(unexpected)
97    catch
98        error:{erpc, timeout} ->
99            ok
100    end,
101    try
102        erpc:call(node(), fun () -> timer:sleep(100) end, 10),
103        ct:fail(unexpected)
104    catch
105        error:{erpc, timeout} ->
106            ok
107    end,
108    {ok, Node} = start_node(Config),
109    call_test(Node, 10000),
110    call_test(Node, infinity),
111    try
112        erpc:call(Node, timer, sleep, [100], 10),
113        ct:fail(unexpected)
114    catch
115        error:{erpc, timeout} ->
116            ok
117    end,
118    try
119        erpc:call(Node, fun () -> timer:sleep(100) end, 10),
120        ct:fail(unexpected)
121    catch
122        error:{erpc, timeout} ->
123            ok
124    end,
125    try
126        erpc:call(Node, erlang, halt, []),
127        ct:fail(unexpected)
128    catch
129        error:{erpc, noconnection} ->
130            ok
131    end,
132    try
133        erpc:call(Node, fun () -> erlang:node() end),
134        ct:fail(unexpected)
135    catch
136        error:{erpc, noconnection} ->
137            ok
138    end,
139    try
140        erpc:call(Node, erlang, node, []),
141        ct:fail(unexpected)
142    catch
143        error:{erpc, noconnection} ->
144            ok
145    end,
146    try
147        erpc:call(badnodename, erlang, node, []),
148        ct:fail(unexpected)
149    catch
150        error:{erpc, noconnection} ->
151            ok
152    end,
153
154    receive after 1000 -> ok end,
155    [] = flush([]),
156    ok.
157
158call_test(Node, Timeout) ->
159    io:format("call_test(~p, ~p)~n", [Node, Timeout]),
160    Node = erpc:call(Node, erlang, node, [], Timeout),
161    try
162        erpc:call(Node, erlang, error, [oops|invalid], Timeout),
163        ct:fail(unexpected)
164    catch
165        error:{exception, badarg, [{erlang,apply,[erlang,error,[oops|invalid]],_}]} ->
166            ok
167    end,
168    try
169        erpc:call(Node, erlang, error, [oops], Timeout),
170        ct:fail(unexpected)
171    catch
172        error:{exception, oops, [{erlang,error,[oops],_}]} ->
173            ok
174    end,
175    try
176        erpc:call(Node, erlang, exit, [oops], Timeout),
177        ct:fail(unexpected)
178    catch
179        exit:{exception, oops} ->
180            ok
181    end,
182    try
183        erpc:call(Node, fun () -> erlang:exit(oops) end, Timeout),
184        ct:fail(unexpected)
185    catch
186        exit:{exception, oops} ->
187            ok
188    end,
189    try
190        erpc:call(Node, erlang, throw, [oops], Timeout),
191        ct:fail(unexpected)
192    catch
193        throw:oops ->
194            ok
195    end,
196    try
197        erpc:call(Node, fun () -> erlang:throw(oops) end, Timeout),
198        ct:fail(unexpected)
199    catch
200        throw:oops ->
201            ok
202    end,
203    case {node() == Node, Timeout == infinity} of
204        {true, true} ->
205            %% This would kill the test since local calls
206            %% without timeout is optimized to execute in
207            %% calling process itself...
208            ok;
209        _ ->
210            ExitSignal = fun () ->
211                                 exit(self(), oops),
212                                 receive after infinity -> ok end
213                         end,
214            try
215                erpc:call(Node, ExitSignal, Timeout),
216                ct:fail(unexpected)
217            catch
218                exit:{signal, oops} ->
219                    ok
220            end,
221            try
222                erpc:call(Node, erlang, apply, [ExitSignal, []], Timeout),
223                ct:fail(unexpected)
224            catch
225                exit:{signal, oops} ->
226                    ok
227            end
228    end,
229    try
230        erpc:call(Node, ?MODULE, call_func1, [boom], Timeout),
231        ct:fail(unexpected)
232    catch
233        error:{exception,
234               {exception,
235                boom,
236                [{?MODULE, call_func3, A2, _},
237                 {?MODULE, call_func2, 1, _}]},
238               [{erpc, call, A1, _},
239                {?MODULE, call_func1, 1, _}]}
240          when ((A1 == 5)
241                orelse (A1 == [Node, ?MODULE, call_func2, [boom]]))
242               andalso ((A2 == 1)
243                        orelse (A2 == [boom])) ->
244            ok
245    end,
246    try
247        erpc:call(Node, fun () -> ?MODULE:call_func1(boom) end, Timeout),
248        ct:fail(unexpected)
249    catch
250        error:{exception,
251               {exception,
252                boom,
253                [{?MODULE, call_func3, A4, _},
254                 {?MODULE, call_func2, 1, _}]},
255               [{erpc, call, A3, _},
256                {?MODULE, call_func1, 1, _},
257                {erlang, apply, 2, _}]}
258          when ((A3 == 5)
259                orelse (A3 == [Node, ?MODULE, call_func2, [boom]]))
260               andalso ((A4 == 1)
261                        orelse (A4 == [boom])) ->
262            ok
263    end,
264    try
265        call_func4(Node, node(), 10, Timeout),
266        ct:fail(unexpected)
267    catch
268        error:Error1 ->
269%%%            io:format("Error1=~p~n", [Error1]),
270            check_call_func4_error(Error1, 10)
271    end,
272    try
273        call_func4(node(), Node, 5, Timeout),
274        ct:fail(unexpected)
275    catch
276        error:Error2 ->
277%%%            io:format("Error2=~p~n", [Error2]),
278            check_call_func4_error(Error2, 5)
279    end,
280    ok.
281
282check_call_func4_error({exception,
283                         badarg,
284                         [{?MODULE, call_func5, _, _},
285                          {?MODULE, call_func4, _, _}]},
286                         0) ->
287    ok;
288check_call_func4_error({exception,
289                        Exception,
290                        [{erpc, call, _, _},
291                         {?MODULE, call_func5, _, _},
292                         {?MODULE, call_func4, _, _}]},
293                       N) ->
294    check_call_func4_error(Exception, N-1).
295
296call_func1(X) ->
297    erpc:call(node(), ?MODULE, call_func2, [X]),
298    ok.
299
300call_func2(X) ->
301    call_func3(X),
302    ok.
303
304call_func3(X) ->
305    erlang:error(X, [X]).
306
307call_func4(A, B, N, T) ->
308    call_func5(A, B, N, T),
309    ok.
310
311call_func5(A, B, N, T) when N >= 0 ->
312    erpc:call(A, ?MODULE, call_func4, [B, A, N-1, T], T),
313    ok;
314call_func5(_A, _B, _N, _T) ->
315    erlang:error(badarg).
316
317call_against_old_node(Config) ->
318    case start_22_node(Config) of
319        {ok, Node22} ->
320            try
321                erpc:call(Node22, erlang, node, []),
322                ct:fail(unexpected)
323            catch
324                error:{erpc, notsup} ->
325                    ok
326            end,
327            stop_node(Node22),
328            ok;
329        _ ->
330	    {skipped, "No OTP 22 available"}
331    end.
332
333call_reqtmo(Config) when is_list(Config) ->
334    Fun = fun (Node, SendMe, Timeout) ->
335                  try
336                      erpc:call(Node, erlang, send,
337                                [self(), SendMe], Timeout),
338                      ct:fail(unexpected)
339                  catch
340                      error:{erpc, timeout} -> ok
341                  end
342          end,
343    reqtmo_test(Config, Fun).
344
345reqtmo_test(Config, Test) ->
346    %% Tests that we time out in time also when the request itself
347    %% does not get through. A typical issue we have had
348    %% in the past, is that the timeout has not triggered until
349    %% the request has gotten through...
350
351    Timeout = 500,
352    WaitBlock = 100,
353    BlockTime = 1000,
354
355    {ok, Node} = start_node(Config),
356
357    erpc:call(Node, erts_debug, set_internal_state, [available_internal_state,
358                                                     true]),
359
360    SendMe = make_ref(),
361
362    erpc:cast(Node, erts_debug, set_internal_state, [block, BlockTime]),
363    receive after WaitBlock -> ok end,
364
365    Start = erlang:monotonic_time(),
366
367    Test(Node, SendMe, Timeout),
368
369    Stop = erlang:monotonic_time(),
370    Time = erlang:convert_time_unit(Stop-Start, native, millisecond),
371    io:format("Actual time: ~p ms~n", [Time]),
372    true = Time >= Timeout,
373    true = Time =< Timeout + 200,
374
375    receive SendMe -> ok end,
376
377    receive UnexpectedMsg -> ct:fail({unexpected_message, UnexpectedMsg})
378    after 0 -> ok
379    end,
380
381    stop_node(Node),
382
383    {comment,
384     "Timeout = " ++ integer_to_list(Timeout)
385     ++ " Actual = " ++ integer_to_list(Time)}.
386
387cast(Config) when is_list(Config) ->
388    %% silently fail
389    ok = erpc:cast(badnodename, erlang, send, [hej]),
390
391    try
392        erpc:cast(<<"node">>, erlang, send, [hej]),
393        ct:fail(unexpected)
394    catch
395        error:{erpc, badarg} -> ok
396    end,
397    try
398        erpc:cast(node(), erlang, send, hej),
399        ct:fail(unexpected)
400    catch
401        error:{erpc, badarg} -> ok
402    end,
403    try
404        erpc:cast(node(), "erlang", send, [hej]),
405        ct:fail(unexpected)
406    catch
407        error:{erpc, badarg} -> ok
408    end,
409    try
410        erpc:cast(node(), erlang, make_ref(), [hej]),
411        ct:fail(unexpected)
412    catch
413        error:{erpc, badarg} -> ok
414    end,
415
416    erpc:cast(node(), erlang, send, [self()|blupp]), %% silent remote error...
417
418    Me = self(),
419    Ok1 = make_ref(),
420    ok = erpc:cast(node(), erlang, send, [Me, {mfa, Ok1}]),
421    receive
422        {mfa, Ok1} -> ok
423    end,
424    ok = erpc:cast(node(), fun () -> Me ! {a_fun, Ok1} end),
425    receive
426        {a_fun, Ok1} -> ok
427    end,
428    {ok, Node} = start_node(Config),
429    Ok2 = make_ref(),
430    ok = erpc:cast(Node, erlang, send, [Me, {mfa, Ok2}]),
431    receive
432        {mfa, Ok2} -> ok
433    end,
434    ok = erpc:cast(Node, fun () -> Me ! {a_fun, Ok2} end),
435    receive
436        {a_fun, Ok2} -> ok
437    end,
438
439    ok = erpc:cast(Node, erlang, halt, []),
440
441    monitor_node(Node, true),
442    receive {nodedown, Node} -> ok end,
443
444    ok = erpc:cast(Node, erlang, send, [Me, wont_reach_me]),
445
446    receive after 1000 -> ok end,
447    [] = flush([]),
448
449    case start_22_node(Config) of
450        {ok, Node22} ->
451            ok = erpc:cast(Node, erlang, send, [Me, wont_reach_me]),
452            ok = erpc:cast(Node, fun () -> Me ! wont_reach_me end),
453            receive
454                Msg -> ct:fail({unexpected_message, Msg})
455            after
456                2000 -> ok
457            end,
458            stop_node(Node22),
459            {comment, "Tested against OTP 22 as well"};
460        _ ->
461            {comment, "No tested against OTP 22"}
462    end.
463
464send_request(Config) when is_list(Config) ->
465    %% Note: First part of nodename sets response delay in seconds.
466    PA = filename:dirname(code:which(?MODULE)),
467    NodeArgs = [{args,"-pa "++ PA}],
468    {ok,Node1} = test_server:start_node('1_erpc_SUITE_call', slave, NodeArgs),
469    {ok,Node2} = test_server:start_node('10_erpc_SUITE_call', slave, NodeArgs),
470    {ok,Node3} = test_server:start_node('20_erpc_SUITE_call', slave, NodeArgs),
471    ReqId1 = erpc:send_request(Node1, ?MODULE, f, []),
472    ReqId2 = erpc:send_request(Node2, ?MODULE, f, []),
473    ReqId3 = erpc:send_request(Node3, ?MODULE, f, []),
474    ReqId4 = erpc:send_request(Node1, erlang, error, [bang]),
475    ReqId5 = erpc:send_request(Node1, ?MODULE, f, []),
476
477    try
478        erpc:receive_response(ReqId4, 1000)
479    catch
480        error:{exception, bang, [{erlang,error,[bang],_}]} ->
481            ok
482    end,
483    try
484        erpc:receive_response(ReqId5, 10)
485    catch
486        error:{erpc, timeout} ->
487            ok
488    end,
489
490    %% Test fast timeouts.
491    no_response = erpc:wait_response(ReqId2),
492    no_response = erpc:wait_response(ReqId2, 10),
493
494    %% Let Node1 finish its work before yielding.
495    ct:sleep({seconds,2}),
496    {hej,_,Node1} = erpc:receive_response(ReqId1),
497
498    %% Wait for the Node2 and Node3.
499    {response,{hej,_,Node2}} = erpc:wait_response(ReqId2, infinity),
500    {hej,_,Node3} = erpc:receive_response(ReqId3),
501
502    try
503        erpc:receive_response(ReqId5, 10),
504        ct:fail(unexpected)
505    catch
506        error:{erpc, badarg} ->
507            ok
508    end,
509
510    receive
511        Msg0 -> ct:fail(Msg0)
512    after 0 -> ok
513    end,
514
515    stop_node(Node1),
516    stop_node(Node2),
517    stop_node(Node3),
518
519    [] = flush([]),
520
521    {ok, Node4} = start_node(Config),
522
523    ReqId6 = erpc:send_request(Node4, erlang, node, []),
524
525    no_response = erpc:check_response({response, Node1}, ReqId6),
526    no_response = erpc:check_response(ReqId6, ReqId6),
527    receive
528        Msg1 ->
529            {response, Node4} = erpc:check_response(Msg1, ReqId6)
530    end,
531
532    ReqId7 = erpc:send_request(Node4, erlang, halt, []),
533    try
534        erpc:receive_response(ReqId7),
535        ct:fail(unexpected)
536    catch
537        error:{erpc, noconnection} -> ok
538    end,
539
540    ReqId8 = erpc:send_request(Node4, erlang, node, []),
541    receive
542        Msg2 ->
543            try
544                erpc:check_response(Msg2, ReqId8),
545                ct:fail(unexpected)
546            catch
547                error:{erpc, noconnection} ->
548                    ok
549            end
550    end,
551
552    [] = flush([]),
553
554    case start_22_node(Config) of
555        {ok, Node5} ->
556            ReqId9 = erpc:send_request(Node5, erlang, node, []),
557            try
558                erpc:receive_response(ReqId9),
559                ct:fail(unexpected)
560            catch
561                error:{erpc, notsup} -> ok
562            end,
563
564            stop_node(Node5),
565
566            [] = flush([]),
567
568            ok;
569        _ ->
570            {comment, "No test against OTP 22 node performed"}
571    end.
572
573send_request_fun(Config) when is_list(Config) ->
574    %% Note: First part of nodename sets response delay in seconds.
575    PA = filename:dirname(code:which(?MODULE)),
576    NodeArgs = [{args,"-pa "++ PA}],
577    {ok,Node1} = test_server:start_node('1_erpc_SUITE_call', slave, NodeArgs),
578    {ok,Node2} = test_server:start_node('10_erpc_SUITE_call', slave, NodeArgs),
579    {ok,Node3} = test_server:start_node('20_erpc_SUITE_call', slave, NodeArgs),
580    ReqId1 = erpc:send_request(Node1, fun () -> ?MODULE:f() end),
581    ReqId2 = erpc:send_request(Node2, fun () -> ?MODULE:f() end),
582    ReqId3 = erpc:send_request(Node3, fun () -> ?MODULE:f() end),
583    ReqId4 = erpc:send_request(Node1, fun () -> erlang:error(bang) end),
584    ReqId5 = erpc:send_request(Node1, fun () -> ?MODULE:f() end),
585
586    try
587        erpc:receive_response(ReqId4, 1000)
588    catch
589        error:{exception, bang, [{?MODULE, _, _, _},
590                                 {erlang,apply,2,_}]} ->
591            ok
592    end,
593    try
594        erpc:receive_response(ReqId5, 10)
595    catch
596        error:{erpc, timeout} ->
597            ok
598    end,
599
600    %% Test fast timeouts.
601    no_response = erpc:wait_response(ReqId2),
602    no_response = erpc:wait_response(ReqId2, 10),
603
604    %% Let Node1 finish its work before yielding.
605    ct:sleep({seconds,2}),
606    {hej,_,Node1} = erpc:receive_response(ReqId1),
607
608    %% Wait for the Node2 and Node3.
609    {response,{hej,_,Node2}} = erpc:wait_response(ReqId2, infinity),
610    {hej,_,Node3} = erpc:receive_response(ReqId3),
611
612    try
613        erpc:receive_response(ReqId5, 10),
614        ct:fail(unexpected)
615    catch
616        error:{erpc, badarg} ->
617            ok
618    end,
619
620    receive
621        Msg0 -> ct:fail(Msg0)
622    after 0 -> ok
623    end,
624
625    stop_node(Node1),
626    stop_node(Node2),
627    stop_node(Node3),
628
629    {ok, Node4} = start_node(Config),
630
631    ReqId6 = erpc:send_request(Node4, fun () -> erlang:node() end),
632
633    no_response = erpc:check_response({response, Node1}, ReqId6),
634    no_response = erpc:check_response(ReqId6, ReqId6),
635    receive
636        Msg1 ->
637            {response, Node4} = erpc:check_response(Msg1, ReqId6)
638    end,
639
640    ReqId7 = erpc:send_request(Node4, erlang, halt, []),
641    try
642        erpc:receive_response(ReqId7),
643        ct:fail(unexpected)
644    catch
645        error:{erpc, noconnection} -> ok
646    end,
647
648    ReqId8 = erpc:send_request(Node4, erlang, node, []),
649    receive
650        Msg2 ->
651            try
652                erpc:check_response(Msg2, ReqId8),
653                ct:fail(unexpected)
654            catch
655                error:{erpc, noconnection} ->
656                    ok
657            end
658    end,
659
660    [] = flush([]),
661
662    case start_22_node(Config) of
663        {ok, Node5} ->
664            ReqId9 = erpc:send_request(Node5, fun () -> erlang:node() end),
665            try
666                erpc:receive_response(ReqId9),
667                ct:fail(unexpected)
668            catch
669                error:{erpc, notsup} -> ok
670            end,
671
672            stop_node(Node5),
673
674            [] = flush([]),
675
676            ok;
677        _ ->
678            {comment, "No test against OTP 22 node performed"}
679    end.
680
681
682send_request_receive_reqtmo(Config) when is_list(Config) ->
683    Fun = fun (Node, SendMe, Timeout) ->
684                  RID = erpc:send_request(Node, erlang, send,
685                                          [self(), SendMe]),
686                  try
687                      erpc:receive_response(RID, Timeout),
688                      ct:fail(unexpected)
689                  catch
690                      error:{erpc, timeout} -> ok
691                  end
692          end,
693    reqtmo_test(Config, Fun).
694
695send_request_wait_reqtmo(Config) when is_list(Config) ->
696    Fun = fun (Node, SendMe, Timeout) ->
697                  RID = erpc:send_request(Node, erlang, send,
698                                          [self(), SendMe]),
699                  no_response = erpc:wait_response(RID, 0),
700                  no_response = erpc:wait_response(RID, Timeout),
701                  %% Cleanup...
702                  try
703                      erpc:receive_response(RID, 0),
704                      ct:fail(unexpected)
705                  catch
706                      error:{erpc, timeout} -> ok
707                  end
708          end,
709    reqtmo_test(Config, Fun).
710
711send_request_check_reqtmo(Config) when is_list(Config) ->
712    Fun = fun (Node, SendMe, Timeout) ->
713                  RID = erpc:send_request(Node, erlang, send,
714                                          [self(), SendMe]),
715                  receive Msg -> erpc:check_response(Msg, RID)
716                  after Timeout -> ok
717                  end,
718                  %% Cleanup...
719                  try
720                      erpc:receive_response(RID, 0),
721                      ct:fail(unexpected)
722                  catch
723                      error:{erpc, timeout} -> ok
724                  end
725          end,
726    reqtmo_test(Config, Fun).
727
728send_request_against_old_node(Config) when is_list(Config) ->
729    case start_22_node(Config) of
730        {ok, Node22} ->
731            RID1 = erpc:send_request(Node22, erlang, node, []),
732            RID2 = erpc:send_request(Node22, erlang, node, []),
733            RID3 = erpc:send_request(Node22, erlang, node, []),
734            try
735                erpc:receive_response(RID1),
736                ct:fail(unexpected)
737            catch
738                error:{erpc, notsup} ->
739                    ok
740            end,
741            try
742                erpc:wait_response(RID2, infinity),
743                ct:fail(unexpected)
744            catch
745                error:{erpc, notsup} ->
746                    ok
747            end,
748            try
749                receive
750                    Msg ->
751                        erpc:check_response(Msg, RID3),
752                        ct:fail(unexpected)
753                end
754            catch
755                error:{erpc, notsup} ->
756                    ok
757            end,
758            stop_node(Node22),
759            ok;
760        _ ->
761	    {skipped, "No OTP 22 available"}
762    end.
763
764multicall(Config) ->
765    {ok, Node1} = start_node(Config),
766    {ok, Node2} = start_node(Config),
767    {Node3, Node3Res} = case start_22_node(Config) of
768                            {ok, N3} ->
769                                {N3, {error, {erpc, notsup}}};
770                            _ ->
771                                {ok, N3} = start_node(Config),
772                                stop_node(N3),
773                                {N3, {error, {erpc, noconnection}}}
774                        end,
775    {ok, Node4} = start_node(Config),
776    {ok, Node5} = start_node(Config),
777    stop_node(Node2),
778
779    ThisNode = node(),
780    Nodes = [ThisNode, Node1, Node2, Node3, Node4, Node5],
781
782    [{ok, ThisNode},
783     {ok, Node1},
784     {error, {erpc, noconnection}},
785     Node3Res,
786     {ok, Node4},
787     {ok, Node5},
788     {error, {erpc, noconnection}}]
789        = erpc:multicall(Nodes ++ [badnodename], erlang, node, []),
790
791    [{ok, ThisNode},
792     {ok, Node1},
793     {error, {erpc, noconnection}},
794     Node3Res,
795     {ok, Node4},
796     {ok, Node5},
797     {error, {erpc, noconnection}}]
798        = erpc:multicall(Nodes ++ [badnodename], fun () -> erlang:node() end),
799
800    try
801        erpc:multicall(Nodes ++ [<<"badnodename">>], erlang, node, []),
802        ct:fail(unexpected)
803    catch
804        error:{erpc, badarg} ->
805            ok
806    end,
807
808    try
809        erpc:multicall([Node1, Node2, Node3, Node4, Node5 | node()], erlang, node, []),
810        ct:fail(unexpected)
811    catch
812        error:{erpc, badarg} ->
813            ok
814    end,
815
816
817    try
818        erpc:multicall(Nodes ++ [<<"badnodename">>], fun () -> erlang:node() end),
819        ct:fail(unexpected)
820    catch
821        error:{erpc, badarg} ->
822            ok
823    end,
824
825    try
826        erpc:multicall(Nodes, fun (X) -> X end),
827        ct:fail(unexpected)
828    catch
829        error:{erpc, badarg} ->
830            ok
831    end,
832
833    [{ok, ThisNode},
834     {ok, Node1},
835     {error, {erpc, noconnection}},
836     Node3Res,
837     {ok, Node4},
838     {ok, Node5}]
839        = erpc:multicall(Nodes, erlang, node, []),
840
841    [{ok, ThisNode},
842     {ok, Node1},
843     {error, {erpc, noconnection}},
844     Node3Res,
845     {ok, Node4},
846     {ok, Node5}]
847        = erpc:multicall(Nodes, erlang, node, [], 60000),
848
849    [{throw, ThisNode},
850     {throw, Node1},
851     {error, {erpc, noconnection}},
852     Node3Res,
853     {throw, Node4},
854     {throw, Node5}]
855        = erpc:multicall(Nodes, fun () -> throw(erlang:node()) end),
856
857    [{throw, ThisNode},
858     {throw, Node1},
859     {error, {erpc, noconnection}},
860     Node3Res,
861     {throw, Node4},
862     {throw, Node5}]
863        = erpc:multicall(Nodes,  fun () -> throw(erlang:node()) end, 60000),
864
865    S0 = erlang:monotonic_time(millisecond),
866    [{error, {erpc, timeout}},
867     {error, {erpc, timeout}},
868     {error, {erpc, noconnection}},
869     Node3Res,
870     {error, {erpc, timeout}},
871     {error, {erpc, timeout}},
872     {error, {erpc, timeout}},
873     {error, {erpc, timeout}},
874     {error, {erpc, noconnection}},
875     Node3Res,
876     {error, {erpc, timeout}},
877     {error, {erpc, timeout}}]
878        = erpc:multicall(Nodes++Nodes, timer, sleep, [2000], 500),
879    E0 = erlang:monotonic_time(millisecond),
880    T0 = E0 - S0,
881    io:format("T0=~p~n", [T0]),
882    true = T0 < 1000,
883
884    S1 = erlang:monotonic_time(millisecond),
885    [{ok, ok},
886     {ok, ok},
887     {error, {erpc, noconnection}},
888     Node3Res,
889     {ok, ok},
890     {ok, ok},
891     {ok, ok},
892     {ok, ok},
893     {error, {erpc, noconnection}},
894     Node3Res,
895     {ok, ok},
896     {ok, ok}]
897        = erpc:multicall(Nodes++Nodes, timer, sleep, [2000]),
898    E1 = erlang:monotonic_time(millisecond),
899    T1 = E1 - S1,
900    io:format("T1=~p~n", [T1]),
901    true = T1 < 3000,
902
903    S2 = erlang:monotonic_time(millisecond),
904    [{ok, ok},
905     {ok, ok},
906     {error, {erpc, noconnection}},
907     Node3Res,
908     {ok, ok},
909     {ok, ok},
910     {ok, ok},
911     {ok, ok},
912     {error, {erpc, noconnection}},
913     Node3Res,
914     {ok, ok},
915     {ok, ok}]
916        = erpc:multicall(Nodes++Nodes, timer, sleep, [2000], 3000),
917    E2 = erlang:monotonic_time(millisecond),
918    T2 = E2 - S2,
919    io:format("T2=~p~n", [T2]),
920    true = T2 < 3000,
921
922    [{ok, ThisNode},
923     {ok, Node1},
924     {error, {erpc, noconnection}},
925     Node3Res,
926     {ok, Node4},
927     {ok, Node5},
928     {ok, ThisNode},
929     {ok, Node1},
930     {error, {erpc, noconnection}},
931     Node3Res,
932     {ok, Node4},
933     {ok, Node5}]
934        = erpc:multicall(Nodes++Nodes, erlang, node, []),
935
936    [{ok, ThisNode},
937     {ok, Node1},
938     {error, {erpc, noconnection}},
939     Node3Res,
940     {ok, Node4},
941     {ok, Node5},
942     {ok, ThisNode},
943     {ok, Node1},
944     {error, {erpc, noconnection}},
945     Node3Res,
946     {ok, Node4},
947     {ok, Node5}]
948        = erpc:multicall(Nodes++Nodes, erlang, node, [], 60000),
949
950    [{ok, ThisNode},
951     {ok, Node1},
952     {error, {erpc, noconnection}},
953     Node3Res,
954     {ok, Node4},
955     {ok, Node5}]
956        = erpc:multicall(Nodes, fun () -> erlang:node() end),
957
958    [BlingError,
959     BlingError,
960     {error, {erpc, noconnection}},
961     Node3Res,
962     BlingError,
963     BlingError]
964        = erpc:multicall(Nodes, ?MODULE, call_func2, [bling]),
965
966    [BlingError,
967     BlingError,
968     {error, {erpc, noconnection}},
969     Node3Res,
970     BlingError,
971     BlingError]
972        = erpc:multicall(Nodes, ?MODULE, call_func2, [bling], 60000),
973
974    {error, {exception,
975             bling,
976             [{?MODULE, call_func3, A, _},
977              {?MODULE, call_func2, 1, _}]}} = BlingError,
978    true = (A == 1) orelse (A == [bling]),
979
980    [{error, {exception, blong, [{erlang, error, [blong], _}]}},
981     {error, {exception, blong, [{erlang, error, [blong], _}]}},
982     {error, {erpc, noconnection}},
983     Node3Res,
984     {error, {exception, blong, [{erlang, error, [blong], _}]}},
985     {error, {exception, blong, [{erlang, error, [blong], _}]}}]
986        = erpc:multicall(Nodes, erlang, error, [blong]),
987
988    [{error, {exception, blong, [{erlang, error, [blong], _}]}},
989     {error, {exception, blong, [{erlang, error, [blong], _}]}},
990     {error, {erpc, noconnection}},
991     Node3Res,
992     {error, {exception, blong, [{erlang, error, [blong], _}]}},
993     {error, {exception, blong, [{erlang, error, [blong], _}]}}]
994        = erpc:multicall(Nodes, erlang, error, [blong], 60000),
995
996    SlowNode4 = fun () ->
997                        case node() of
998                            Node4 ->
999                                receive after 1000 -> ok end,
1000                                slow;
1001                            ThisNode ->
1002                                throw(fast);
1003                            _ ->
1004                                fast
1005                           end
1006                end,
1007
1008    Start1 = erlang:monotonic_time(),
1009    [{throw, fast},
1010     {ok, fast},
1011     {error, {erpc, noconnection}},
1012     Node3Res,
1013     {error, {erpc, timeout}},
1014     {ok, fast}]
1015        = erpc:multicall(Nodes, erlang, apply, [SlowNode4, []], 500),
1016
1017    End1 = erlang:monotonic_time(),
1018
1019    Time1 = erlang:convert_time_unit(End1-Start1, native, millisecond),
1020    io:format("Time1 = ~p~n",[Time1]),
1021    true = Time1 >= 500,
1022    true = Time1 =< 1000,
1023
1024    SlowThisNode = fun () ->
1025                           case node() of
1026                               ThisNode ->
1027                                   receive after 1000 -> ok end,
1028                                   slow;
1029                               Node4 ->
1030                                   throw(fast);
1031                               Node5 ->
1032                                   exit(fast);
1033                               _ ->
1034                                   fast
1035                           end
1036                   end,
1037
1038    Start2 = erlang:monotonic_time(),
1039    [{error, {erpc, timeout}},
1040     {ok, fast},
1041     {error, {erpc, noconnection}},
1042     Node3Res,
1043     {throw, fast},
1044     {exit, {exception, fast}}] = erpc:multicall(Nodes, SlowThisNode, 500),
1045
1046    End2 = erlang:monotonic_time(),
1047
1048    Time2 = erlang:convert_time_unit(End2-Start2, native, millisecond),
1049    io:format("Time2 = ~p~n",[Time2]),
1050    true = Time2 >= 500,
1051    true = Time2 =< 1000,
1052
1053    %% We should not get any stray messages due to timed out operations...
1054    receive Msg -> ct:fail({unexpected, Msg})
1055    after 1000 -> ok
1056    end,
1057
1058    [{error, {erpc, noconnection}},
1059     Node3Res,
1060     {error, {erpc, noconnection}},
1061     {error, {erpc, noconnection}}]
1062        = erpc:multicall([Node2, Node3, Node4, Node5], erlang, halt, []),
1063
1064    [] = flush([]),
1065
1066    stop_node(Node3),
1067    case Node3Res of
1068        {error, {erpc, notsup}} ->
1069            {comment, "Tested against an OTP 22 node as well"};
1070        _ ->
1071            {comment, "No OTP 22 node available; i.e., only testing against current release"}
1072    end.
1073
1074multicall_reqtmo(Config) when is_list(Config) ->
1075    {ok, QuickNode1} = start_node(Config),
1076    {ok, QuickNode2} = start_node(Config),
1077    Fun = fun (Node, SendMe, Timeout) ->
1078                  Me = self(),
1079                  SlowSend = fun () ->
1080                                     if node() == Node ->
1081                                             Me ! SendMe,
1082                                             done;
1083                                        true ->
1084                                             done
1085                                     end
1086                             end,
1087                  [{ok, done},{error,{erpc,timeout}},{ok, done}]
1088                      = erpc:multicall([QuickNode1, Node, QuickNode2],
1089                                        erlang, apply, [SlowSend, []],
1090                                        Timeout)
1091          end,
1092    Res = reqtmo_test(Config, Fun),
1093    stop_node(QuickNode1),
1094    stop_node(QuickNode2),
1095    Res.
1096
1097multicall_recv_opt(Config) when is_list(Config) ->
1098    Loops = 1000,
1099    HugeMsgQ = 500000,
1100    process_flag(message_queue_data, off_heap),
1101    {ok, Node1} = start_node(Config),
1102    {ok, Node2} = start_node(Config),
1103    ExpRes = [{ok, node()}, {ok, Node1}, {ok, Node2}],
1104    Nodes = [node(), Node1, Node2],
1105    Fun = fun () -> erlang:node() end,
1106    _Warmup = time_multicall(ExpRes, Nodes, Fun, infinity, Loops div 10),
1107    Empty = time_multicall(ExpRes, Nodes, Fun, infinity, Loops),
1108    io:format("Time with empty message queue: ~p microsecond~n",
1109	      [erlang:convert_time_unit(Empty, native, microsecond)]),
1110    _ = [self() ! {msg,N} || N <- lists:seq(1, HugeMsgQ)],
1111    Huge = time_multicall(ExpRes, Nodes, Fun, infinity, Loops),
1112    io:format("Time with huge message queue: ~p microsecond~n",
1113	      [erlang:convert_time_unit(Huge, native, microsecond)]),
1114    stop_node(Node1),
1115    stop_node(Node2),
1116    Q = Huge / Empty,
1117    HugeMsgQ = flush_msgq(),
1118    case Q > 10 of
1119	true ->
1120	    ct:fail({ratio, Q});
1121	false ->
1122	    {comment, "Ratio: "++erlang:float_to_list(Q)}
1123    end.
1124
1125multicall_recv_opt2(Config) when is_list(Config) ->
1126    Loops = 1000,
1127    HugeMsgQ = 500000,
1128    process_flag(message_queue_data, off_heap),
1129    {ok, Node1} = start_node(Config),
1130    stop_node(Node1),
1131    {ok, Node2} = start_node(Config),
1132    ExpRes = [{ok, node()}, {error, {erpc, noconnection}}, {ok, Node2}],
1133    Nodes = [node(), Node1, Node2],
1134    Fun = fun () -> erlang:node() end,
1135    _Warmup = time_multicall(ExpRes, Nodes, Fun, infinity, Loops div 10),
1136    Empty = time_multicall(ExpRes, Nodes, Fun, infinity, Loops),
1137    io:format("Time with empty message queue: ~p microsecond~n",
1138	      [erlang:convert_time_unit(Empty, native, microsecond)]),
1139    _ = [self() ! {msg,N} || N <- lists:seq(1, HugeMsgQ)],
1140    Huge = time_multicall(ExpRes, Nodes, Fun, infinity, Loops),
1141    io:format("Time with huge message queue: ~p microsecond~n",
1142	      [erlang:convert_time_unit(Huge, native, microsecond)]),
1143    stop_node(Node2),
1144    Q = Huge / Empty,
1145    HugeMsgQ = flush_msgq(),
1146    case Q > 10 of
1147	true ->
1148	    ct:fail({ratio, Q});
1149	false ->
1150	    {comment, "Ratio: "++erlang:float_to_list(Q)}
1151    end.
1152
1153multicall_recv_opt3(Config) when is_list(Config) ->
1154    Loops = 1000,
1155    HugeMsgQ = 500000,
1156    process_flag(message_queue_data, off_heap),
1157    {ok, Node1} = start_node(Config),
1158    stop_node(Node1),
1159    {ok, Node2} = start_node(Config),
1160    Nodes = [node(), Node1, Node2],
1161    Fun = fun () -> erlang:node() end,
1162    _Warmup = time_multicall(undefined, Nodes, Fun, infinity, Loops div 10),
1163    Empty = time_multicall(undefined, Nodes, Fun, infinity, Loops),
1164    io:format("Time with empty message queue: ~p microsecond~n",
1165	      [erlang:convert_time_unit(Empty, native, microsecond)]),
1166    _ = [self() ! {msg,N} || N <- lists:seq(1, HugeMsgQ)],
1167    Huge = time_multicall(undefined, Nodes, Fun, 0, Loops),
1168    io:format("Time with huge message queue: ~p microsecond~n",
1169	      [erlang:convert_time_unit(Huge, native, microsecond)]),
1170    stop_node(Node2),
1171    Q = Huge / Empty,
1172    HugeMsgQ = flush_msgq(),
1173    case Q > 10 of
1174	true ->
1175	    ct:fail({ratio, Q});
1176	false ->
1177	    {comment, "Ratio: "++erlang:float_to_list(Q)}
1178    end.
1179
1180time_multicall(Expect, Nodes, Fun, Tmo, Times) ->
1181    Start = erlang:monotonic_time(),
1182    ok = do_time_multicall(Expect, Nodes, Fun, Tmo, Times),
1183    erlang:monotonic_time() - Start.
1184
1185do_time_multicall(_Expect, _Nodes, _Fun, _Tmo, 0) ->
1186    ok;
1187do_time_multicall(undefined, Nodes, Fun, Tmo, N) ->
1188    _ = erpc:multicall(Nodes, Fun, Tmo),
1189    do_time_multicall(undefined, Nodes, Fun, Tmo, N-1);
1190do_time_multicall(Expect, Nodes, Fun, Tmo, N) ->
1191    Expect = erpc:multicall(Nodes, Fun, Tmo),
1192    do_time_multicall(Expect, Nodes, Fun, Tmo, N-1).
1193
1194multicast(Config) when is_list(Config) ->
1195    {ok, Node} = start_node(Config),
1196    Nodes = [node(), Node],
1197    try
1198        erpc:multicast(Nodes, erlang, send, hej),
1199        ct:fail(unexpected)
1200    catch
1201        error:{erpc, badarg} -> ok
1202    end,
1203    try
1204        erpc:multicast(node(), erlang, send, [hej]),
1205        ct:fail(unexpected)
1206    catch
1207        error:{erpc, badarg} -> ok
1208    end,
1209    try
1210        erpc:multicast([<<"node">>, Node], erlang, send, [hej]),
1211        ct:fail(unexpected)
1212    catch
1213        error:{erpc, badarg} -> ok
1214    end,
1215    try
1216        erpc:multicast(Nodes, "erlang", send, [hej]),
1217        ct:fail(unexpected)
1218    catch
1219        error:{erpc, badarg} -> ok
1220    end,
1221    try
1222        erpc:multicast(Nodes, erlang, make_ref(), [hej]),
1223        ct:fail(unexpected)
1224    catch
1225        error:{erpc, badarg} -> ok
1226    end,
1227
1228    %% Silently fail...
1229    erpc:multicast([badnodename], erlang, send, [self(), blupp]),
1230    %% silent remote error...
1231    erpc:multicast(Nodes, erlang, send, [self()|blupp]),
1232
1233    Me = self(),
1234    Ok1 = make_ref(),
1235    ok = erpc:multicast(Nodes, erlang, send, [Me, {mfa, Ok1}]),
1236    receive
1237        {mfa, Ok1} -> ok
1238    end,
1239    receive
1240        {mfa, Ok1} -> ok
1241    end,
1242    ok = erpc:multicast(Nodes, fun () -> Me ! {a_fun, Ok1} end),
1243    receive
1244        {a_fun, Ok1} -> ok
1245    end,
1246    receive
1247        {a_fun, Ok1} -> ok
1248    end,
1249
1250    ok = erpc:multicast([Node], erlang, halt, []),
1251
1252    monitor_node(Node, true),
1253    receive {nodedown, Node} -> ok end,
1254
1255    ok = erpc:multicast([Node], erlang, send, [Me, wont_reach_me]),
1256
1257    receive after 1000 -> ok end,
1258
1259    [] = flush([]),
1260    case start_22_node(Config) of
1261        {ok, Node22} ->
1262            ok = erpc:multicast([Node], erlang, send, [Me, wont_reach_me]),
1263            ok = erpc:multicast([Node], fun () -> Me ! wont_reach_me end),
1264            receive
1265                Msg -> ct:fail({unexpected_message, Msg})
1266            after
1267                2000 -> ok
1268            end,
1269            stop_node(Node22),
1270            {comment, "Tested against OTP 22 as well"};
1271        _ ->
1272            {comment, "No tested against OTP 22"}
1273    end.
1274
1275timeout_limit(Config) when is_list(Config) ->
1276    Node = node(),
1277    MaxTmo = (1 bsl 32) - 1,
1278    erlang:send_after(100, self(), dummy_message),
1279    try
1280        receive
1281            M ->
1282                M
1283        after MaxTmo + 1 ->
1284                ok
1285        end,
1286        ct:fail("The ?MAX_INT_TIMEOUT define in erpc.erl needs "
1287                "to be updated to reflect max timeout value "
1288                "in a receive/after...")
1289    catch
1290        error:timeout_value ->
1291            ok
1292    end,
1293    Node = erpc:call(Node, erlang, node, [], MaxTmo),
1294    try
1295        erpc:call(node(), erlang, node, [], MaxTmo+1),
1296        ct:fail(unexpected)
1297    catch
1298        error:{erpc, badarg} ->
1299            ok
1300    end,
1301    [{ok,Node}] = erpc:multicall([Node], erlang, node, [], MaxTmo),
1302    try
1303        erpc:multicall([Node], erlang, node, [], MaxTmo+1),
1304        ct:fail(unexpected)
1305    catch
1306        error:{erpc, badarg} ->
1307            ok
1308    end,
1309    ReqId1 = erpc:send_request(Node, erlang, node, []),
1310    Node = erpc:receive_response(ReqId1, MaxTmo),
1311    ReqId2 = erpc:send_request(Node, erlang, node, []),
1312    try
1313        erpc:receive_response(ReqId2, MaxTmo+1),
1314        ct:fail(unexpected)
1315    catch
1316        error:{erpc, badarg} ->
1317            ok
1318    end,
1319    ReqId3 = erpc:send_request(Node, erlang, node, []),
1320    Node = erpc:receive_response(ReqId3, MaxTmo),
1321    ReqId4 = erpc:send_request(Node, erlang, node, []),
1322    try
1323        erpc:receive_response(ReqId4, MaxTmo+1),
1324        ct:fail(unexpected)
1325    catch
1326        error:{erpc, badarg} ->
1327            ok
1328    end,
1329    ok.
1330
1331
1332%%%
1333%%% Utility functions.
1334%%%
1335
1336start_node(Config) ->
1337    Name = list_to_atom(atom_to_list(?MODULE)
1338			++ "-" ++ atom_to_list(proplists:get_value(testcase, Config))
1339			++ "-" ++ integer_to_list(erlang:system_time(second))
1340			++ "-" ++ integer_to_list(erlang:unique_integer([positive]))),
1341    Pa = filename:dirname(code:which(?MODULE)),
1342    test_server:start_node(Name, slave, [{args,  "-pa " ++ Pa}]).
1343
1344start_22_node(Config) ->
1345    Rel = "22_latest",
1346    case test_server:is_release_available(Rel) of
1347	false ->
1348            notsup;
1349        true ->
1350            Cookie = atom_to_list(erlang:get_cookie()),
1351            Name = list_to_atom(atom_to_list(?MODULE)
1352                                ++ "-" ++ atom_to_list(proplists:get_value(testcase, Config))
1353                                ++ "-" ++ integer_to_list(erlang:system_time(second))
1354                                ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))),
1355            Pa = filename:dirname(code:which(?MODULE)),
1356	    test_server:start_node(Name,
1357                                   peer,
1358                                   [{args, "-pa " ++ Pa ++ " -setcookie "++Cookie},
1359                                    {erl, [{release, Rel}]}])
1360    end.
1361
1362stop_node(Node) ->
1363    test_server:stop_node(Node).
1364
1365flush(L) ->
1366    receive
1367	M ->
1368	    flush([M|L])
1369    after 0 ->
1370	    L
1371    end.
1372
1373t() ->
1374    [N | _] = string:tokens(atom_to_list(node()), "_"),
1375    1000*list_to_integer(N).
1376
1377f() ->
1378    timer:sleep(T=t()),
1379    spawn(?MODULE, f2, []),
1380    {hej,T,node()}.
1381
1382f2() ->
1383    timer:sleep(500),
1384    halt().
1385
1386flush_msgq() ->
1387    flush_msgq(0).
1388flush_msgq(N) ->
1389    receive
1390	_ ->
1391	    flush_msgq(N+1)
1392    after 0 ->
1393	    N
1394    end.
1395