1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2003-2021. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20
21%%
22%%----------------------------------------------------------------------
23%% Purpose: Verify the application specifics of the Megaco application
24%%----------------------------------------------------------------------
25-module(megaco_load_SUITE).
26
27-export([
28 	 suite/0, all/0, groups/0,
29         init_per_suite/1, end_per_suite/1,
30         init_per_group/2, end_per_group/2,
31         init_per_testcase/2, end_per_testcase/2,
32
33         single_user_light_load/1,
34         single_user_medium_load/1,
35         single_user_heavy_load/1,
36         single_user_extreme_load/1,
37
38         multi_user_light_load/1,
39         multi_user_medium_load/1,
40         multi_user_heavy_load/1,
41         multi_user_extreme_load/1
42
43        ]).
44
45-export([
46         do_multi_load/3,
47         multi_load_collector/7
48        ]).
49
50
51-include_lib("common_test/include/ct.hrl").
52-include_lib("megaco/include/megaco.hrl").
53-include_lib("megaco/include/megaco_message_v1.hrl").
54-include("megaco_test_lib.hrl").
55
56-define(TEST_VERBOSITY, debug).
57-define(MGC_VERBOSITY,  silence).
58-define(MG_VERBOSITY,   silence).
59
60-define(SINGLE_USER_LOAD_NUM_REQUESTS, 1000).
61-define(MULTI_USER_LOAD_NUM_REQUESTS,  1000).
62
63-define(MGC_START(Node, Mid, ET, Conf, Verb),
64        megaco_test_mgc:start(Node, Mid, ET,
65			      Conf ++ [{megaco_trace, false}], Verb)).
66-define(MGC_STOP(Pid), megaco_test_mgc:stop(Pid)).
67-define(MGC_USER_INFO(Pid,Tag), megaco_test_mgc:user_info(Pid,Tag)).
68-define(MGC_CONN_INFO(Pid,Tag), megaco_test_mgc:conn_info(Pid,Tag)).
69-define(MGC_SET_VERBOSITY(Pid, V), megaco_test_mgc:verbosity(Pid, V)).
70
71-define(MG_START(Pid, Mid, Enc, Transp, Conf, Verb),
72        megaco_test_mg:start(Pid, Mid, Enc, Transp,
73			     Conf ++ [{megaco_trace, false},
74                                      {transport_opts, [{serialize, true}]}],
75			     Verb)).
76-define(MG_STOP(Pid), megaco_test_mg:stop(Pid)).
77-define(MG_USER_INFO(Pid,Tag), megaco_test_mg:user_info(Pid,Tag)).
78-define(MG_CONN_INFO(Pid,Tag), megaco_test_mg:conn_info(Pid,Tag)).
79-define(MG_SERV_CHANGE(Pid), megaco_test_mg:service_change(Pid)).
80-define(MG_MLOAD(Pid, NL, NR),
81	timer:tc(megaco_test_mg, apply_multi_load, [Pid, NL, NR])).
82-define(MG_LOAD(Pid, NL, NR), megaco_test_mg:apply_multi_load(Pid, NL, NR)).
83-define(MG_SET_VERBOSITY(Pid, V), megaco_test_mg:verbosity(Pid, V)).
84
85
86
87%%======================================================================
88%% Common Test interface functions
89%%======================================================================
90
91suite() ->
92    [{ct_hooks, [ts_install_cth]}].
93
94all() ->
95    %% This is a temporary messure to ensure that we can
96    %% test the socket backend without effecting *all*
97    %% applications on *all* machines.
98    %% This flag is set only for *one* host.
99    case ?TEST_INET_BACKENDS() of
100        true ->
101            [
102             {group, inet_backend_default},
103             {group, inet_backend_inet},
104             {group, inet_backend_socket}
105            ];
106        _ ->
107            [
108             {group, inet_backend_default}
109            ]
110    end.
111
112groups() ->
113    [
114     {inet_backend_default, [], inet_backend_default_cases()},
115     {inet_backend_inet,    [], inet_backend_inet_cases()},
116     {inet_backend_socket,  [], inet_backend_socket_cases()},
117
118     {all,                  [], all_cases()},
119     {single,               [], single_cases()},
120     {multi,                [], multi_cases()}
121    ].
122
123inet_backend_default_cases() ->
124    [{all, [], all_cases()}].
125
126inet_backend_inet_cases() ->
127    [{all, [], all_cases()}].
128
129inet_backend_socket_cases() ->
130    [{all, [], all_cases()}].
131
132all_cases() ->
133    [
134     {group, single},
135     {group, multi}
136    ].
137
138single_cases() ->
139    [
140     single_user_light_load,
141     single_user_medium_load,
142     single_user_heavy_load,
143     single_user_extreme_load
144    ].
145
146multi_cases() ->
147    [
148     multi_user_light_load,
149     multi_user_medium_load,
150     multi_user_heavy_load,
151     multi_user_extreme_load
152    ].
153
154
155
156%%
157%% -----
158%%
159
160init_per_suite(suite) ->
161    [];
162init_per_suite(doc) ->
163    [];
164init_per_suite(Config0) when is_list(Config0) ->
165
166    ?ANNOUNCE_SUITE_INIT(),
167
168    p("init_per_suite -> entry with"
169      "~n      Config: ~p"
170      "~n      Nodes:  ~p", [Config0, erlang:nodes()]),
171
172    case ?LIB:init_per_suite(Config0) of
173        {skip, _} = SKIP ->
174            SKIP;
175
176        Config1 when is_list(Config1) ->
177
178            %% We need a (local) monitor on this node also
179            megaco_test_sys_monitor:start(),
180
181            p("init_per_suite -> end when"
182              "~n      Config: ~p"
183              "~n      Nodes:  ~p", [Config1, erlang:nodes()]),
184
185            Config1
186    end.
187
188end_per_suite(suite) -> [];
189end_per_suite(doc) -> [];
190end_per_suite(Config0) when is_list(Config0) ->
191
192    p("end_per_suite -> entry with"
193      "~n      Config: ~p"
194      "~n      Nodes:  ~p", [Config0, erlang:nodes()]),
195
196    megaco_test_sys_monitor:stop(),
197    Config1 = ?LIB:end_per_suite(Config0),
198
199    p("end_per_suite -> end when"
200      "~n      Nodes:  ~p", [erlang:nodes()]),
201
202    Config1.
203
204
205%%
206%% -----
207%%
208
209init_per_group(inet_backend_default = Group, Config) ->
210    ?ANNOUNCE_GROUP_INIT(Group),
211    [{socket_create_opts, []} | Config];
212init_per_group(inet_backend_inet = Group, Config) ->
213    ?ANNOUNCE_GROUP_INIT(Group),
214    case ?EXPLICIT_INET_BACKEND() of
215        true ->
216            %% The environment trumps us,
217            %% so only the default group should be run!
218            {skip, "explicit inet backend"};
219        false ->
220            [{socket_create_opts, [{inet_backend, inet}]} | Config]
221    end;
222init_per_group(inet_backend_socket = Group, Config) ->
223    ?ANNOUNCE_GROUP_INIT(Group),
224    case ?EXPLICIT_INET_BACKEND() of
225        true ->
226            %% The environment trumps us,
227            %% so only the default group should be run!
228            {skip, "explicit inet backend"};
229        false ->
230            [{socket_create_opts, [{inet_backend, socket}]} | Config]
231    end;
232init_per_group(Group, Config) ->
233    ?ANNOUNCE_GROUP_INIT(Group),
234    Config.
235
236end_per_group(Group, Config) when (inet_backend_default =:= Group) orelse
237                                  (inet_backend_init    =:= Group) orelse
238                                  (inet_backend_socket  =:= Group) ->
239    ?SLEEP(?SECS(5)),
240    Config;
241end_per_group(_Group, Config) ->
242    Config.
243
244
245
246%%
247%% -----
248%%
249
250init_per_testcase(single_user_light_load = Case, Config) ->
251    C = lists:keydelete(tc_timeout, 1, Config),
252    do_init_per_testcase(Case, [{tc_timeout, min(2)}|C]);
253init_per_testcase(single_user_medium_load = Case, Config) ->
254    C = lists:keydelete(tc_timeout, 1, Config),
255    do_init_per_testcase(Case, [{tc_timeout, min(5)}|C]);
256init_per_testcase(single_user_heavy_load = Case, Config) ->
257    C = lists:keydelete(tc_timeout, 1, Config),
258    do_init_per_testcase(Case, [{tc_timeout, min(10)}|C]);
259init_per_testcase(single_user_extreme_load = Case, Config) ->
260    C = lists:keydelete(tc_timeout, 1, Config),
261    do_init_per_testcase(Case, [{tc_timeout, min(20)}|C]);
262init_per_testcase(multi_user_light_load = Case, Config) ->
263    C = lists:keydelete(tc_timeout, 1, Config),
264    do_init_per_testcase(Case, [{tc_timeout, min(2)}|C]);
265init_per_testcase(multi_user_medium_load = Case, Config) ->
266    C = lists:keydelete(tc_timeout, 1, Config),
267    do_init_per_testcase(Case, [{tc_timeout, min(5)}|C]);
268init_per_testcase(multi_user_heavy_load = Case, Config) ->
269    C = lists:keydelete(tc_timeout, 1, Config),
270    do_init_per_testcase(Case, [{tc_timeout, min(10)}|C]);
271init_per_testcase(multi_user_extreme_load = Case, Config) ->
272    C = lists:keydelete(tc_timeout, 1, Config),
273    do_init_per_testcase(Case, [{tc_timeout, min(20)}|C]);
274init_per_testcase(Case, Config) ->
275    do_init_per_testcase(Case, Config).
276
277do_init_per_testcase(Case, Config) ->
278    process_flag(trap_exit, true),
279
280    p("init_per_suite -> entry with"
281      "~n      Config: ~p"
282      "~n      Nodes:  ~p", [Config, erlang:nodes()]),
283
284    megaco_test_global_sys_monitor:reset_events(),
285    megaco_test_lib:init_per_testcase(Case, Config).
286
287end_per_testcase(Case, Config) ->
288    process_flag(trap_exit, false),
289
290    p("end_per_suite -> entry with"
291      "~n      Config: ~p"
292      "~n      Nodes:  ~p", [Config, erlang:nodes()]),
293
294    p("system events during test: "
295      "~n   ~p", [megaco_test_global_sys_monitor:events()]),
296
297    megaco_test_lib:end_per_testcase(Case, Config).
298
299
300
301%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
302
303single_user_light_load(suite) ->
304    [];
305single_user_light_load(doc) ->
306    [];
307single_user_light_load(Config) when is_list(Config) ->
308    try_single_user_load(single_user_light_load, Config, 5).
309
310
311%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
312
313single_user_medium_load(suite) ->
314    [];
315single_user_medium_load(doc) ->
316    [];
317single_user_medium_load(Config) when is_list(Config) ->
318    try_single_user_load(single_user_medium_load, Config, 15).
319
320
321%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
322
323single_user_heavy_load(suite) ->
324    [];
325single_user_heavy_load(doc) ->
326    [];
327single_user_heavy_load(Config) when is_list(Config) ->
328    try_single_user_load(single_user_heavy_load, Config, 25).
329
330
331%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
332
333single_user_extreme_load(suite) ->
334    [];
335single_user_extreme_load(doc) ->
336    [];
337single_user_extreme_load(Config) when is_list(Config) ->
338    try_single_user_load(single_user_extreme_load, Config, 100).
339
340
341%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
342
343multi_user_light_load(suite) ->
344    [];
345multi_user_light_load(doc) ->
346    [];
347multi_user_light_load(Config) when is_list(Config) ->
348    try_multi_user_load(multi_user_light_load, Config, 3, 1).
349
350
351%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
352
353multi_user_medium_load(suite) ->
354    [];
355multi_user_medium_load(doc) ->
356    [];
357multi_user_medium_load(Config) when is_list(Config) ->
358    try_multi_user_load(multi_user_medium_load, Config, 3, 5).
359
360
361%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
362
363multi_user_heavy_load(suite) ->
364    [];
365multi_user_heavy_load(doc) ->
366    [];
367multi_user_heavy_load(Config) when is_list(Config) ->
368    try_multi_user_load(multi_user_heavy_load, Config, 3, 10).
369
370
371%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
372
373multi_user_extreme_load(suite) ->
374    [];
375multi_user_extreme_load(doc) ->
376    [];
377multi_user_extreme_load(Config) when is_list(Config) ->
378    try_multi_user_load(multi_user_extreme_load, Config, 3, 15).
379
380
381%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
382
383populate([]) ->
384    ok;
385populate([{Key,Val}|Env]) ->
386    put(Key, Val),
387    populate(Env).
388
389load_controller(Config, Fun) when is_list(Config) and is_function(Fun) ->
390    process_flag(trap_exit, true),
391    {value, {tc_timeout, TcTimeout}} =
392	lists:keysearch(tc_timeout, 1, Config),
393    SkipTimeout = trunc(95*TcTimeout/100), % 95% of TcTimeout
394    Env = get(),
395    Loader = erlang:spawn_link(fun() -> Fun(Env) end),
396    receive
397	{'EXIT', Loader, normal} ->
398	    d("load_controller -> "
399	      "loader [~p] terminated with normal", [Loader]),
400	    ok;
401	{'EXIT', Loader, ok} ->
402	    d("load_controller -> "
403	      "loader [~p] terminated with ok~n", [Loader]),
404	    ok;
405	{'EXIT', Loader, {skipped, {fatal, Reason, File, Line}}} ->
406	    i("load_controller -> "
407	      "loader [~p] terminated with fatal skip"
408	      "~n   Reason: ~p"
409	      "~n   At:     ~p:~p", [Loader, Reason, File, Line]),
410	    ?SKIP(Reason);
411	{'EXIT', Loader, {skipped, Reason}} ->
412	    i("load_controller -> "
413	      "loader [~p] terminated with skip"
414	      "~n   Reason: ~p", [Loader, Reason]),
415	    ?SKIP(Reason);
416	{'EXIT', Loader, Reason} ->
417	    i("load_controller -> "
418	      "loader [~p] terminated with"
419	      "~n   ~p", [Loader, Reason]),
420	    erlang:error({unexpected_loader_result, Reason})
421    after SkipTimeout ->
422	    i("load_controller -> "
423	      "loader [~p] timeout", [Loader]),
424	    exit(Loader, kill),
425	    ?SKIP({timeout, SkipTimeout, TcTimeout})
426    end.
427
428%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
429
430try_single_user_load(Name, Config, NumLoaders0) ->
431    Factor     = ?config(megaco_factor, Config),
432    NumLoaders =
433	if
434	    (Factor =:= 1) ->
435		NumLoaders0;
436	    (Factor > NumLoaders0) ->
437		1;
438	    true ->
439		NumLoaders0 div Factor
440	end,
441    Pre = fun() ->
442		  MgcNode = make_node_name(mgc),
443		  MgNode  = make_node_name(mg),
444		  d("Nodes: "
445		    "~n      MgcNode: ~p"
446		    "~n      MgNode:  ~p", [MgcNode, MgNode]),
447		  Nodes = [MgcNode, MgNode],
448		  ok = ?START_NODES(Nodes),
449		  Nodes
450	  end,
451    Case = fun(State) -> single_user_load(State, Config, NumLoaders) end,
452    Post = fun(Nodes) ->
453                   d("stop nodes"),
454                   ?STOP_NODES(lists:reverse(Nodes))
455           end,
456    try_tc(Name, Pre, Case, Post).
457
458
459single_user_load(State, Config, NumLoaders) ->
460    i("starting with ~w loader(s)", [NumLoaders]),
461    SCO = ?config(socket_create_opts, Config),
462    Res = load_controller(Config,
463			  fun(Env) ->
464				  populate(Env),
465				  exit( do_single_user_load(SCO,
466                                                            State, NumLoaders) )
467			  end),
468    i("done"),
469    Res.
470
471do_single_user_load(SCO,
472                    [MgcNode, MgNode], NumLoaders) ->
473    %% Start the MGC and MGs
474    i("[MGC] start"),
475    MgcMid = {deviceName, "ctrl"},
476    ET     = [{text, tcp, [{serialize, true}]}],
477    DSI = maybe_display_system_info(NumLoaders),
478    {ok, Mgc} = ?MGC_START(MgcNode, MgcMid, ET, SCO ++ DSI, ?MGC_VERBOSITY),
479
480    i("[MG] start"),
481    MgMid = {deviceName, "mg"},
482    {ok, Mg} = ?MG_START(MgNode, MgMid, text, tcp, SCO ++ DSI, ?MG_VERBOSITY),
483
484    d("MG user info: ~p", [?MG_USER_INFO(Mg, all)]),
485
486    i("[MG] connect to the MGC (service change)"),
487    ServChRes = ?MG_SERV_CHANGE(Mg),
488    d("service change result: ~p", [ServChRes]),
489
490    megaco_test_mg:update_conn_info(Mg,reply_timer,1000),
491    megaco_test_mgc:update_conn_info(Mgc,reply_timer,1000),
492
493    d("MG conn info: ~p", [?MG_CONN_INFO(Mg, all)]),
494
495    d("apply the load"),
496    Res = ?MG_MLOAD(Mg, NumLoaders, ?SINGLE_USER_LOAD_NUM_REQUESTS),
497    case Res of
498	{Time, {ok, Ok, Err}} ->
499	    Sec = Time / 1000000,
500	    io:format("~nmultiple loaders result: ~n"
501		      "   Number of successfull: ~w~n"
502		      "   Number of failure:     ~w~n"
503		      "   Time:                  ~w seconds~n"
504		      "   Calls / seconds        ~w~n~n",
505		      [Ok, Err, Sec, (NumLoaders * ?SINGLE_USER_LOAD_NUM_REQUESTS)/Sec]);
506	{Time, Error} ->
507	    io:format("SUL: multiple loaders failed: ~p after ~w~n",
508		      [Error, Time])
509    end,
510
511    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
512
513    i("verbosity to trace"),
514    ?MGC_SET_VERBOSITY(Mgc, debug),
515    ?MG_SET_VERBOSITY(Mg, debug),
516
517    %% Tell MG to stop
518    i("[MG] stop"),
519    ?MG_STOP(Mg),
520
521    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
522
523    %% Tell Mgc to stop
524    i("[MGC] stop"),
525    ?MGC_STOP(Mgc),
526
527    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
528
529    ok.
530
531
532
533%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
534
535try_multi_user_load(Name, Config, NumUsers, NumLoaders0) ->
536    Factor     = ?config(megaco_factor, Config),
537    NumLoaders =
538	if
539	    (Factor =:= 1) ->
540		NumLoaders0;
541	    (Factor > NumLoaders0) ->
542		1;
543	    true ->
544		NumLoaders0 div Factor
545	end,
546    Pre = fun() ->
547		  MgcNode = make_node_name(mgc),
548		  MgNodes = make_node_names(mg, NumUsers),
549		  d("Nodes: "
550		    "~n      MgcNode: ~p"
551		    "~n      MgNodes: ~p", [MgcNode, MgNodes]),
552		  Nodes = [MgcNode | MgNodes],
553		  ok = ?START_NODES(Nodes),
554		  Nodes
555	  end,
556    Case = fun(State) ->
557		   multi_user_load(State, Config, NumUsers, NumLoaders)
558	   end,
559    Post = fun(Nodes) ->
560                   d("stop nodes"),
561                   ?STOP_NODES(lists:reverse(Nodes))
562           end,
563    try_tc(Name, Pre, Case, Post).
564
565
566multi_user_load(State, Config, NumUsers, NumLoaders) ->
567    i("starting with ~w loader(s)", [NumLoaders]),
568    SCO = ?config(socket_create_opts, Config),
569    Res = load_controller(
570	    Config,
571	    fun(Env) ->
572		    populate(Env),
573		    exit( do_multi_user_load(SCO, State, NumUsers, NumLoaders) )
574	    end),
575    i("done"),
576    Res.
577
578
579do_multi_user_load(SCO, [MgcNode | MgNodes], NumUsers, NumLoaders)
580  when (is_integer(NumUsers) andalso (NumUsers > 1) andalso
581	is_integer(NumLoaders) andalso (NumLoaders >= 1)) ->
582    %% Start the MGC and MGs
583    i("[MGC] start"),
584    MgcMid = {deviceName, "ctrl"},
585    ET     = [{text, tcp, [{serialize, false}]}],
586    DSI = maybe_display_system_info(2 * NumUsers * NumLoaders),
587    {ok, Mgc} = ?MGC_START(MgcNode, MgcMid, ET, SCO ++ DSI, ?MGC_VERBOSITY),
588
589    megaco_test_mgc:update_user_info(Mgc,reply_timer,1000),
590    d("MGC user info: ~p", [?MGC_USER_INFO(Mgc, all)]),
591
592    MgUsers = make_mids(MgNodes),
593
594    d("start MGs, apply the load and stop MGs"),
595    ok = multi_load(MgUsers, SCO ++ DSI, NumLoaders, ?MULTI_USER_LOAD_NUM_REQUESTS),
596
597    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
598
599    ?MGC_SET_VERBOSITY(Mgc, debug),
600
601    %% Tell Mgc to stop
602    i("[MGC] stop"),
603    ?MGC_STOP(Mgc),
604
605    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
606
607    ok.
608
609
610multi_load(MGs, Conf, NumLoaders, NumReqs) ->
611    d("multi_load -> entry with"
612      "~n   MGs:        ~p"
613      "~n   Conf:       ~p"
614      "~n   NumLoaders: ~p"
615      "~n   NumReqs:    ~p", [MGs, Conf, NumLoaders, NumReqs]),
616
617    Pids = multi_load_collector_start(MGs, Conf, NumLoaders, NumReqs, []),
618    case timer:tc(?MODULE, do_multi_load, [Pids, NumLoaders, NumReqs]) of
619	{Time, {ok, OKs, []}} ->
620	    Sec = Time / 1000000,
621	    multi_load_collector_calc(Sec, OKs);
622	{Time, Error} ->
623	    Sec = Time/1000000,
624	    io:format("~nmulti load failed after ~.1f:~n~p~n~n", [Sec,Error]),
625	    {error, Error}
626    end.
627
628do_multi_load(Pids, _NumLoaders, _NumReqs) ->
629    Fun =
630	fun({P,_}) ->
631		d("apply multi load for ~p", [P]),
632		P ! {apply_multi_load, self()}
633	end,
634    lists:foreach(Fun, Pids),
635    await_multi_load_collectors(Pids, [], []).
636
637multi_load_collector_start([], _Conf, _NumLoaders, _NumReqs, Pids) ->
638    Pids;
639multi_load_collector_start([{Mid, Node}|MGs], Conf, NumLoaders, NumReqs, Pids) ->
640    Env = get(),
641    Pid = spawn_link(?MODULE, multi_load_collector,
642		     [self(), Node, Mid, Conf, NumLoaders, NumReqs, Env]),
643    multi_load_collector_start(MGs, Conf, NumLoaders, NumReqs, [{Pid,Mid}|Pids]).
644
645get_env(Key, Env) ->
646    case lists:keysearch(Key, 1, Env) of
647	{value, {Key, Val}} ->
648	    Val;
649	_ ->
650	    undefined
651    end.
652
653multi_load_collector(Parent, Node, Mid, Conf, NumLoaders, NumReqs, Env) ->
654    put(verbosity, get_env(verbosity, Env)),
655    put(tc, get_env(tc, Env)),
656    put(sname, get_env(sname, Env) ++ "-loader"),
657    case ?MG_START(Node, Mid, text, tcp, Conf, ?MG_VERBOSITY) of
658	{ok, Pid} ->
659	    d("MG ~p user info: ~n~p", [Mid, ?MG_USER_INFO(Pid, all)]),
660	    ServChRes = ?MG_SERV_CHANGE(Pid),
661	    d("service change result: ~p", [ServChRes]),
662	    megaco_test_mg:update_conn_info(Pid,reply_timer,1000),
663	    d("MG ~p conn info: ~p", [Mid, ?MG_CONN_INFO(Pid,all)]),
664	    multi_load_collector_loop(Parent, Pid, Mid, NumLoaders, NumReqs);
665	Else ->
666	    Parent ! {load_start_failed, self(), Mid, Else}
667    end.
668
669multi_load_collector_loop(Parent, Pid, Mid, NumLoaders, NumReqs) ->
670    d("multi_load_collector_loop -> entry with"
671      "~n   Parent:     ~p"
672      "~n   Pid:        ~p"
673      "~n   Mid:        ~p"
674      "~n   NumLoaders: ~p"
675      "~n   NumReqs:    ~p"
676      "~nwhen"
677      "~n   self():     ~p"
678      "~n   node():     ~p",
679      [Parent, Pid, Mid, NumLoaders, NumReqs, self(), node()]),
680    receive
681	{apply_multi_load, Parent} ->
682	    Res = ?MG_LOAD(Pid, NumLoaders, NumReqs),
683	    Parent ! {load_complete, self(), Mid, Res},
684	    ?MG_SET_VERBOSITY(Pid, debug),
685	    ?MG_STOP(Pid),
686	    exit(normal)
687    end.
688
689
690await_multi_load_collectors([], Oks, Errs) ->
691    i("await_multi_load_collectors -> done"),
692    {ok, Oks, Errs};
693await_multi_load_collectors(Pids, Oks, Errs) ->
694    receive
695	{load_complete, Pid, Mg, {ok, Ok, Err}} ->
696	    i("await_multi_load_collectors -> "
697	      "received ok complete from "
698	      "~n   ~p [~p]", [Pid, Mg]),
699	    Pids2 = lists:keydelete(Pid, 1, Pids),
700	    Oks2  = [{Mg, Ok, Err}|Oks],
701	    await_multi_load_collectors(Pids2, Oks2, Errs);
702	{load_complete, Pid, Mg, Error} ->
703	    i("await_multi_load_collectors -> "
704	      "received error complete from "
705	      "~n   ~p [~p]: "
706	      "~n   ~p", [Pid, Mg, Error]),
707	    Pids2 = lists:keydelete(Pid, 1, Pids),
708	    Errs2 = [{Mg, Error}|Errs],
709	    await_multi_load_collectors(Pids2, Oks, Errs2);
710
711	{'EXIT', Pid, normal} ->
712	    %% This is assumed to be one of the collectors
713	    i("await_multi_load_collectors -> "
714	      "received (normal) exit signal from ~p", [Pid]),
715	    await_multi_load_collectors(Pids, Oks, Errs);
716
717	{'EXIT', Pid, Reason} ->
718	    i("await_multi_load_collectors -> "
719	      "received unexpected exit from ~p:"
720	      "~n   ~p", [Pid, Reason]),
721	    case lists:keydelete(Pid, 1, Pids) of
722		Pids ->
723		    %% Not one of my procs, or a proc I have already
724		    %% received a complete from.
725		    await_multi_load_collectors(Pids, Oks, Errs);
726		Pids2 ->
727		    [{Pid,Mg}] = Pids -- Pids2,
728		    Errs2 = [{Mg, {unexpected_exit, Reason}}|Errs],
729		    await_multi_load_collectors(Pids, Oks, Errs2)
730	    end;
731
732	Else ->
733	    i("await_multi_load_collectors -> received unexpected message:"
734	      "~n~p", [Else]),
735	    await_multi_load_collectors(Pids, Oks, Errs)
736    after
737	5000 ->
738	    i("await_multi_load_collectors -> still awaiting reply from:"
739	      "~n~p", [Pids]),
740	    await_multi_load_collectors(Pids, Oks, Errs)
741    end.
742
743
744%% Note that this is an approximation...we run all the
745%% MGs in parrallel, so it should be "accurate"...
746multi_load_collector_calc(Sec, Oks) ->
747    Succs = lists:sum([Ok   || {_, Ok,   _} <- Oks]),
748    Fails = lists:sum([Err  || {_,  _, Err} <- Oks]),
749    io:format("~ntotal multiple loaders result: ~n"
750	      "   Number of successfull: ~w~n"
751	      "   Number of failure:     ~w~n"
752	      "   Total Calls / seconds: ~.2f~n~n",
753	      [Succs, Fails, Sec]),
754    ok.
755
756%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
757
758make_node_names(Name, Num) ->
759    make_node_names(Name, Num, []).
760
761make_node_names(_, 0, Names) ->
762    Names;
763make_node_names(BaseName, N, Names) ->
764    Name = lists:flatten(io_lib:format("~p~w", [BaseName,N])),
765    make_node_names(BaseName, N-1, [make_node_name(Name)|Names]).
766
767make_node_name(Name) when is_atom(Name) ->
768    make_node_name(atom_to_list(Name));
769make_node_name(Name) when is_list(Name) ->
770    case string:tokens(atom_to_list(node()), [$@]) of
771	[_,Host] ->
772	    list_to_atom(lists:concat([Name ++ "@" ++ Host]));
773	_ ->
774	    exit("Test node must be started with '-sname'")
775     end.
776
777make_mids(MgNodes) when is_list(MgNodes) andalso (length(MgNodes) > 0) ->
778    make_mids(MgNodes, []).
779
780make_mids([], Mids) ->
781    lists:reverse(Mids);
782make_mids([MgNode|MgNodes], Mids) ->
783    case string:tokens(atom_to_list(MgNode), [$@]) of
784	[Name, _] ->
785	    Mid = {deviceName, Name},
786	    make_mids(MgNodes, [{Mid, MgNode}|Mids]);
787	_Else ->
788	    exit("Test node must be started with '-sname'")
789    end.
790
791maybe_display_system_info(NumLoaders) when NumLoaders > 50 ->
792    [{display_system_info, timer:seconds(2)}];
793maybe_display_system_info(NumLoaders) when NumLoaders > 10 ->
794    [{display_system_info, timer:seconds(1)}];
795maybe_display_system_info(_) ->
796    [].
797
798
799%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
800
801try_tc(TCName, Pre, Case, Post) ->
802    try_tc(TCName, "TEST", ?TEST_VERBOSITY, Pre, Case, Post).
803
804try_tc(TCName, Name, Verbosity, Pre, Case, Post) ->
805    ?TRY_TC(TCName, Name, Verbosity, Pre, Case, Post).
806
807
808%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
809
810min(M) -> timer:minutes(M).
811
812p(F, A) ->
813    io:format("*** [~s] ~p ***"
814	      "~n   " ++ F ++ "~n",
815	      [?FTS(), self() | A]).
816
817i(F) ->
818    i(F, []).
819
820i(F, A) ->
821    print(info, get(verbosity), get(tc), "INF", F, A).
822
823d(F) ->
824    d(F, []).
825
826d(F, A) ->
827    print(debug, get(verbosity), get(tc), "DBG", F, A).
828
829printable(_, debug)   -> true;
830printable(info, info) -> true;
831printable(_,_)        -> false.
832
833print(Severity, Verbosity, Tc, P, F, A) ->
834    print(printable(Severity,Verbosity), Tc, P, F, A).
835
836print(true, Tc, P, F, A) ->
837    io:format("*** [~s] ~s ~p ~s:~w ***"
838              "~n   " ++ F ++ "~n",
839              [?FTS(), P, self(), get(sname), Tc | A]);
840print(_, _, _, _, _) ->
841    ok.
842
843
844