1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2003-2020. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20
21%%
22%%----------------------------------------------------------------------
23%% Purpose: Verify the application specifics of the Megaco application
24%%----------------------------------------------------------------------
25-module(megaco_load_SUITE).
26
27-export([
28 	 suite/0, all/0, groups/0,
29         init_per_suite/1, end_per_suite/1,
30         init_per_group/2, end_per_group/2,
31         init_per_testcase/2, end_per_testcase/2,
32
33         single_user_light_load/1,
34         single_user_medium_load/1,
35         single_user_heavy_load/1,
36         single_user_extreme_load/1,
37
38         multi_user_light_load/1,
39         multi_user_medium_load/1,
40         multi_user_heavy_load/1,
41         multi_user_extreme_load/1
42
43        ]).
44
45-export([
46         do_multi_load/3,
47         multi_load_collector/7
48        ]).
49
50
51-include_lib("common_test/include/ct.hrl").
52-include_lib("megaco/include/megaco.hrl").
53-include_lib("megaco/include/megaco_message_v1.hrl").
54-include("megaco_test_lib.hrl").
55
56-define(TEST_VERBOSITY, debug).
57-define(MGC_VERBOSITY,  silence).
58-define(MG_VERBOSITY,   silence).
59
60-define(SINGLE_USER_LOAD_NUM_REQUESTS, 1000).
61-define(MULTI_USER_LOAD_NUM_REQUESTS,  1000).
62
63-define(MGC_START(Pid, Mid, ET, Conf, Verb),
64        megaco_test_mgc:start(Pid, Mid, ET,
65			      [{megaco_trace, false}] ++ Conf, Verb)).
66-define(MGC_STOP(Pid), megaco_test_mgc:stop(Pid)).
67-define(MGC_USER_INFO(Pid,Tag), megaco_test_mgc:user_info(Pid,Tag)).
68-define(MGC_CONN_INFO(Pid,Tag), megaco_test_mgc:conn_info(Pid,Tag)).
69-define(MGC_SET_VERBOSITY(Pid, V), megaco_test_mgc:verbosity(Pid, V)).
70
71-define(MG_START(Pid, Mid, Enc, Transp, Conf, Verb),
72        megaco_test_mg:start(Pid, Mid, Enc, Transp,
73			     [{megaco_trace, false},
74			      {transport_opts, [{serialize, true}]}] ++ Conf,
75			     Verb)).
76-define(MG_STOP(Pid), megaco_test_mg:stop(Pid)).
77-define(MG_USER_INFO(Pid,Tag), megaco_test_mg:user_info(Pid,Tag)).
78-define(MG_CONN_INFO(Pid,Tag), megaco_test_mg:conn_info(Pid,Tag)).
79-define(MG_SERV_CHANGE(Pid), megaco_test_mg:service_change(Pid)).
80-define(MG_MLOAD(Pid, NL, NR),
81	timer:tc(megaco_test_mg, apply_multi_load, [Pid, NL, NR])).
82-define(MG_LOAD(Pid, NL, NR), megaco_test_mg:apply_multi_load(Pid, NL, NR)).
83-define(MG_SET_VERBOSITY(Pid, V), megaco_test_mg:verbosity(Pid, V)).
84
85
86
87%%======================================================================
88%% Common Test interface functions
89%%======================================================================
90
91suite() ->
92    [{ct_hooks, [ts_install_cth]}].
93
94all() ->
95    [
96     {group, single},
97     {group, multi}
98    ].
99
100groups() ->
101    [
102     {single, [], single_cases()},
103     {multi,  [], multi_cases()}
104    ].
105
106single_cases() ->
107    [
108     single_user_light_load,
109     single_user_medium_load,
110     single_user_heavy_load,
111     single_user_extreme_load
112    ].
113
114multi_cases() ->
115    [
116     multi_user_light_load,
117     multi_user_medium_load,
118     multi_user_heavy_load,
119     multi_user_extreme_load
120    ].
121
122
123
124%%
125%% -----
126%%
127
128init_per_suite(suite) ->
129    [];
130init_per_suite(doc) ->
131    [];
132init_per_suite(Config0) when is_list(Config0) ->
133
134    ?ANNOUNCE_SUITE_INIT(),
135
136    p("init_per_suite -> entry with"
137      "~n      Config: ~p"
138      "~n      Nodes:  ~p", [Config0, erlang:nodes()]),
139
140    case ?LIB:init_per_suite(Config0) of
141        {skip, _} = SKIP ->
142            SKIP;
143
144        Config1 when is_list(Config1) ->
145
146            %% We need a (local) monitor on this node also
147            megaco_test_sys_monitor:start(),
148
149            p("init_per_suite -> end when"
150              "~n      Config: ~p"
151              "~n      Nodes:  ~p", [Config1, erlang:nodes()]),
152
153            Config1
154    end.
155
156end_per_suite(suite) -> [];
157end_per_suite(doc) -> [];
158end_per_suite(Config0) when is_list(Config0) ->
159
160    p("end_per_suite -> entry with"
161      "~n      Config: ~p"
162      "~n      Nodes:  ~p", [Config0, erlang:nodes()]),
163
164    megaco_test_sys_monitor:stop(),
165    Config1 = ?LIB:end_per_suite(Config0),
166
167    p("end_per_suite -> end when"
168      "~n      Nodes:  ~p", [erlang:nodes()]),
169
170    Config1.
171
172
173%%
174%% -----
175%%
176
177init_per_group(_GroupName, Config) ->
178    Config.
179
180end_per_group(_GroupName, Config) ->
181    Config.
182
183
184
185%%
186%% -----
187%%
188
189init_per_testcase(single_user_light_load = Case, Config) ->
190    C = lists:keydelete(tc_timeout, 1, Config),
191    do_init_per_testcase(Case, [{tc_timeout, min(2)}|C]);
192init_per_testcase(single_user_medium_load = Case, Config) ->
193    C = lists:keydelete(tc_timeout, 1, Config),
194    do_init_per_testcase(Case, [{tc_timeout, min(5)}|C]);
195init_per_testcase(single_user_heavy_load = Case, Config) ->
196    C = lists:keydelete(tc_timeout, 1, Config),
197    do_init_per_testcase(Case, [{tc_timeout, min(10)}|C]);
198init_per_testcase(single_user_extreme_load = Case, Config) ->
199    C = lists:keydelete(tc_timeout, 1, Config),
200    do_init_per_testcase(Case, [{tc_timeout, min(20)}|C]);
201init_per_testcase(multi_user_light_load = Case, Config) ->
202    C = lists:keydelete(tc_timeout, 1, Config),
203    do_init_per_testcase(Case, [{tc_timeout, min(2)}|C]);
204init_per_testcase(multi_user_medium_load = Case, Config) ->
205    C = lists:keydelete(tc_timeout, 1, Config),
206    do_init_per_testcase(Case, [{tc_timeout, min(5)}|C]);
207init_per_testcase(multi_user_heavy_load = Case, Config) ->
208    C = lists:keydelete(tc_timeout, 1, Config),
209    do_init_per_testcase(Case, [{tc_timeout, min(10)}|C]);
210init_per_testcase(multi_user_extreme_load = Case, Config) ->
211    C = lists:keydelete(tc_timeout, 1, Config),
212    do_init_per_testcase(Case, [{tc_timeout, min(20)}|C]);
213init_per_testcase(Case, Config) ->
214    do_init_per_testcase(Case, Config).
215
216do_init_per_testcase(Case, Config) ->
217    process_flag(trap_exit, true),
218
219    p("init_per_suite -> entry with"
220      "~n      Config: ~p"
221      "~n      Nodes:  ~p", [Config, erlang:nodes()]),
222
223    megaco_test_global_sys_monitor:reset_events(),
224    megaco_test_lib:init_per_testcase(Case, Config).
225
226end_per_testcase(Case, Config) ->
227    process_flag(trap_exit, false),
228
229    p("end_per_suite -> entry with"
230      "~n      Config: ~p"
231      "~n      Nodes:  ~p", [Config, erlang:nodes()]),
232
233    p("system events during test: "
234      "~n   ~p", [megaco_test_global_sys_monitor:events()]),
235
236    megaco_test_lib:end_per_testcase(Case, Config).
237
238
239
240%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
241
242single_user_light_load(suite) ->
243    [];
244single_user_light_load(doc) ->
245    [];
246single_user_light_load(Config) when is_list(Config) ->
247    try_single_user_load(single_user_light_load, Config, 5).
248
249
250%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
251
252single_user_medium_load(suite) ->
253    [];
254single_user_medium_load(doc) ->
255    [];
256single_user_medium_load(Config) when is_list(Config) ->
257    try_single_user_load(single_user_medium_load, Config, 15).
258
259
260%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
261
262single_user_heavy_load(suite) ->
263    [];
264single_user_heavy_load(doc) ->
265    [];
266single_user_heavy_load(Config) when is_list(Config) ->
267    try_single_user_load(single_user_heavy_load, Config, 25).
268
269
270%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
271
272single_user_extreme_load(suite) ->
273    [];
274single_user_extreme_load(doc) ->
275    [];
276single_user_extreme_load(Config) when is_list(Config) ->
277    try_single_user_load(single_user_extreme_load, Config, 100).
278
279
280%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
281
282multi_user_light_load(suite) ->
283    [];
284multi_user_light_load(doc) ->
285    [];
286multi_user_light_load(Config) when is_list(Config) ->
287    try_multi_user_load(multi_user_light_load, Config, 3, 1).
288
289
290%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
291
292multi_user_medium_load(suite) ->
293    [];
294multi_user_medium_load(doc) ->
295    [];
296multi_user_medium_load(Config) when is_list(Config) ->
297    try_multi_user_load(multi_user_medium_load, Config, 3, 5).
298
299
300%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
301
302multi_user_heavy_load(suite) ->
303    [];
304multi_user_heavy_load(doc) ->
305    [];
306multi_user_heavy_load(Config) when is_list(Config) ->
307    try_multi_user_load(multi_user_heavy_load, Config, 3, 10).
308
309
310%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
311
312multi_user_extreme_load(suite) ->
313    [];
314multi_user_extreme_load(doc) ->
315    [];
316multi_user_extreme_load(Config) when is_list(Config) ->
317    try_multi_user_load(multi_user_extreme_load, Config, 3, 15).
318
319
320%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
321
322populate([]) ->
323    ok;
324populate([{Key,Val}|Env]) ->
325    put(Key, Val),
326    populate(Env).
327
328load_controller(Config, Fun) when is_list(Config) and is_function(Fun) ->
329    process_flag(trap_exit, true),
330    {value, {tc_timeout, TcTimeout}} =
331	lists:keysearch(tc_timeout, 1, Config),
332    SkipTimeout = trunc(95*TcTimeout/100), % 95% of TcTimeout
333    Env = get(),
334    Loader = erlang:spawn_link(fun() -> Fun(Env) end),
335    receive
336	{'EXIT', Loader, normal} ->
337	    d("load_controller -> "
338	      "loader [~p] terminated with normal", [Loader]),
339	    ok;
340	{'EXIT', Loader, ok} ->
341	    d("load_controller -> "
342	      "loader [~p] terminated with ok~n", [Loader]),
343	    ok;
344	{'EXIT', Loader, {skipped, {fatal, Reason, File, Line}}} ->
345	    i("load_controller -> "
346	      "loader [~p] terminated with fatal skip"
347	      "~n   Reason: ~p"
348	      "~n   At:     ~p:~p", [Loader, Reason, File, Line]),
349	    ?SKIP(Reason);
350	{'EXIT', Loader, {skipped, Reason}} ->
351	    i("load_controller -> "
352	      "loader [~p] terminated with skip"
353	      "~n   Reason: ~p", [Loader, Reason]),
354	    ?SKIP(Reason);
355	{'EXIT', Loader, Reason} ->
356	    i("load_controller -> "
357	      "loader [~p] terminated with"
358	      "~n   ~p", [Loader, Reason]),
359	    erlang:error({unexpected_loader_result, Reason})
360    after SkipTimeout ->
361	    i("load_controller -> "
362	      "loader [~p] timeout", [Loader]),
363	    exit(Loader, kill),
364	    ?SKIP({timeout, SkipTimeout, TcTimeout})
365    end.
366
367%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
368
369try_single_user_load(Name, Config, NumLoaders0) ->
370    Factor     = ?config(megaco_factor, Config),
371    NumLoaders =
372	if
373	    (Factor =:= 1) ->
374		NumLoaders0;
375	    (Factor > NumLoaders0) ->
376		1;
377	    true ->
378		NumLoaders0 div Factor
379	end,
380    Pre = fun() ->
381		  MgcNode = make_node_name(mgc),
382		  MgNode  = make_node_name(mg),
383		  d("Nodes: "
384		    "~n      MgcNode: ~p"
385		    "~n      MgNode:  ~p", [MgcNode, MgNode]),
386		  Nodes = [MgcNode, MgNode],
387		  ok = ?START_NODES(Nodes),
388		  Nodes
389	  end,
390    Case = fun(State) -> single_user_load(State, Config, NumLoaders) end,
391    Post = fun(Nodes) ->
392                   d("stop nodes"),
393                   ?STOP_NODES(lists:reverse(Nodes))
394           end,
395    try_tc(Name, Pre, Case, Post).
396
397
398single_user_load(State, Config, NumLoaders) ->
399    i("starting with ~w loader(s)", [NumLoaders]),
400    Res = load_controller(Config,
401			  fun(Env) ->
402				  populate(Env),
403				  exit( single_user_load(State, NumLoaders) )
404			  end),
405    i("done"),
406    Res.
407
408single_user_load([MgcNode, MgNode], NumLoaders) ->
409    %% Start the MGC and MGs
410    i("[MGC] start"),
411    MgcMid = {deviceName, "ctrl"},
412    ET     = [{text, tcp, [{serialize, true}]}],
413    DSI = maybe_display_system_info(NumLoaders),
414    {ok, Mgc} = ?MGC_START(MgcNode, MgcMid, ET, DSI, ?MGC_VERBOSITY),
415
416    i("[MG] start"),
417    MgMid = {deviceName, "mg"},
418    {ok, Mg} = ?MG_START(MgNode, MgMid, text, tcp, DSI, ?MG_VERBOSITY),
419
420    d("MG user info: ~p", [?MG_USER_INFO(Mg, all)]),
421
422    i("[MG] connect to the MGC (service change)"),
423    ServChRes = ?MG_SERV_CHANGE(Mg),
424    d("service change result: ~p", [ServChRes]),
425
426    megaco_test_mg:update_conn_info(Mg,reply_timer,1000),
427    megaco_test_mgc:update_conn_info(Mgc,reply_timer,1000),
428
429    d("MG conn info: ~p", [?MG_CONN_INFO(Mg, all)]),
430
431    d("apply the load"),
432    Res = ?MG_MLOAD(Mg, NumLoaders, ?SINGLE_USER_LOAD_NUM_REQUESTS),
433    case Res of
434	{Time, {ok, Ok, Err}} ->
435	    Sec = Time / 1000000,
436	    io:format("~nmultiple loaders result: ~n"
437		      "   Number of successfull: ~w~n"
438		      "   Number of failure:     ~w~n"
439		      "   Time:                  ~w seconds~n"
440		      "   Calls / seconds        ~w~n~n",
441		      [Ok, Err, Sec, (NumLoaders * ?SINGLE_USER_LOAD_NUM_REQUESTS)/Sec]);
442	{Time, Error} ->
443	    io:format("SUL: multiple loaders failed: ~p after ~w~n",
444		      [Error, Time])
445    end,
446
447    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
448
449    i("verbosity to trace"),
450    ?MGC_SET_VERBOSITY(Mgc, debug),
451    ?MG_SET_VERBOSITY(Mg, debug),
452
453    %% Tell MG to stop
454    i("[MG] stop"),
455    ?MG_STOP(Mg),
456
457    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
458
459    %% Tell Mgc to stop
460    i("[MGC] stop"),
461    ?MGC_STOP(Mgc),
462
463    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
464
465    ok.
466
467
468
469%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
470
471try_multi_user_load(Name, Config, NumUsers, NumLoaders0) ->
472    Factor     = ?config(megaco_factor, Config),
473    NumLoaders =
474	if
475	    (Factor =:= 1) ->
476		NumLoaders0;
477	    (Factor > NumLoaders0) ->
478		1;
479	    true ->
480		NumLoaders0 div Factor
481	end,
482    Pre = fun() ->
483		  MgcNode = make_node_name(mgc),
484		  MgNodes = make_node_names(mg, NumUsers),
485		  d("Nodes: "
486		    "~n      MgcNode: ~p"
487		    "~n      MgNodes: ~p", [MgcNode, MgNodes]),
488		  Nodes = [MgcNode | MgNodes],
489		  ok = ?START_NODES(Nodes),
490		  Nodes
491	  end,
492    Case = fun(State) ->
493		   multi_user_load(State, Config, NumUsers, NumLoaders)
494	   end,
495    Post = fun(Nodes) ->
496                   d("stop nodes"),
497                   ?STOP_NODES(lists:reverse(Nodes))
498           end,
499    try_tc(Name, Pre, Case, Post).
500
501
502multi_user_load(State, Config, NumUsers, NumLoaders) ->
503    i("starting with ~w loader(s)", [NumLoaders]),
504    Res = load_controller(
505	    Config,
506	    fun(Env) ->
507		    populate(Env),
508		    exit( multi_user_load(State, NumUsers, NumLoaders) )
509	    end),
510    i("done"),
511    Res.
512
513
514multi_user_load([MgcNode | MgNodes], NumUsers, NumLoaders)
515  when (is_integer(NumUsers) andalso (NumUsers > 1) andalso
516	is_integer(NumLoaders) andalso (NumLoaders >= 1)) ->
517    %% Start the MGC and MGs
518    i("[MGC] start"),
519    MgcMid = {deviceName, "ctrl"},
520    ET     = [{text, tcp, [{serialize, false}]}],
521    DSI = maybe_display_system_info(2 * NumUsers * NumLoaders),
522    {ok, Mgc} = ?MGC_START(MgcNode, MgcMid, ET, DSI, ?MGC_VERBOSITY),
523
524    megaco_test_mgc:update_user_info(Mgc,reply_timer,1000),
525    d("MGC user info: ~p", [?MGC_USER_INFO(Mgc, all)]),
526
527    MgUsers = make_mids(MgNodes),
528
529    d("start MGs, apply the load and stop MGs"),
530    ok = multi_load(MgUsers, DSI, NumLoaders, ?MULTI_USER_LOAD_NUM_REQUESTS),
531
532    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
533
534    ?MGC_SET_VERBOSITY(Mgc, debug),
535
536    %% Tell Mgc to stop
537    i("[MGC] stop"),
538    ?MGC_STOP(Mgc),
539
540    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
541
542    ok.
543
544
545multi_load(MGs, DSI, NumLoaders, NumReqs) ->
546    d("multi_load -> entry with"
547      "~n   MGs:        ~p"
548      "~n   DSI:        ~p"
549      "~n   NumLoaders: ~p"
550      "~n   NumReqs:    ~p", [MGs, DSI, NumLoaders, NumReqs]),
551
552    Pids = multi_load_collector_start(MGs, DSI, NumLoaders, NumReqs, []),
553    case timer:tc(?MODULE, do_multi_load, [Pids, NumLoaders, NumReqs]) of
554	{Time, {ok, OKs, []}} ->
555	    Sec = Time / 1000000,
556	    multi_load_collector_calc(Sec, OKs);
557	{Time, Error} ->
558	    Sec = Time/1000000,
559	    io:format("~nmulti load failed after ~.1f:~n~p~n~n", [Sec,Error]),
560	    {error, Error}
561    end.
562
563do_multi_load(Pids, _NumLoaders, _NumReqs) ->
564    Fun =
565	fun({P,_}) ->
566		d("apply multi load for ~p", [P]),
567		P ! {apply_multi_load, self()}
568	end,
569    lists:foreach(Fun, Pids),
570    await_multi_load_collectors(Pids, [], []).
571
572multi_load_collector_start([], _DSI, _NumLoaders, _NumReqs, Pids) ->
573    Pids;
574multi_load_collector_start([{Mid, Node}|MGs], DSI, NumLoaders, NumReqs, Pids) ->
575    Env = get(),
576    Pid = spawn_link(?MODULE, multi_load_collector,
577		     [self(), Node, Mid, DSI, NumLoaders, NumReqs, Env]),
578    multi_load_collector_start(MGs, DSI, NumLoaders, NumReqs, [{Pid,Mid}|Pids]).
579
580get_env(Key, Env) ->
581    case lists:keysearch(Key, 1, Env) of
582	{value, {Key, Val}} ->
583	    Val;
584	_ ->
585	    undefined
586    end.
587
588multi_load_collector(Parent, Node, Mid, DSI, NumLoaders, NumReqs, Env) ->
589    put(verbosity, get_env(verbosity, Env)),
590    put(tc, get_env(tc, Env)),
591    put(sname, get_env(sname, Env) ++ "-loader"),
592    case ?MG_START(Node, Mid, text, tcp, DSI, ?MG_VERBOSITY) of
593	{ok, Pid} ->
594	    d("MG ~p user info: ~n~p", [Mid, ?MG_USER_INFO(Pid,all)]),
595	    ServChRes = ?MG_SERV_CHANGE(Pid),
596	    d("service change result: ~p", [ServChRes]),
597	    megaco_test_mg:update_conn_info(Pid,reply_timer,1000),
598	    d("MG ~p conn info: ~p", [Mid, ?MG_CONN_INFO(Pid,all)]),
599	    multi_load_collector_loop(Parent, Pid, Mid, NumLoaders, NumReqs);
600	Else ->
601	    Parent ! {load_start_failed, self(), Mid, Else}
602    end.
603
604multi_load_collector_loop(Parent, Pid, Mid, NumLoaders, NumReqs) ->
605    d("multi_load_collector_loop -> entry with"
606      "~n   Parent:     ~p"
607      "~n   Pid:        ~p"
608      "~n   Mid:        ~p"
609      "~n   NumLoaders: ~p"
610      "~n   NumReqs:    ~p"
611      "~nwhen"
612      "~n   self():     ~p"
613      "~n   node():     ~p",
614      [Parent, Pid, Mid, NumLoaders, NumReqs, self(), node()]),
615    receive
616	{apply_multi_load, Parent} ->
617	    Res = ?MG_LOAD(Pid, NumLoaders, NumReqs),
618	    Parent ! {load_complete, self(), Mid, Res},
619	    ?MG_SET_VERBOSITY(Pid, debug),
620	    ?MG_STOP(Pid),
621	    exit(normal)
622    end.
623
624
625await_multi_load_collectors([], Oks, Errs) ->
626    i("await_multi_load_collectors -> done"),
627    {ok, Oks, Errs};
628await_multi_load_collectors(Pids, Oks, Errs) ->
629    receive
630	{load_complete, Pid, Mg, {ok, Ok, Err}} ->
631	    i("await_multi_load_collectors -> "
632	      "received ok complete from "
633	      "~n   ~p [~p]", [Pid, Mg]),
634	    Pids2 = lists:keydelete(Pid, 1, Pids),
635	    Oks2  = [{Mg, Ok, Err}|Oks],
636	    await_multi_load_collectors(Pids2, Oks2, Errs);
637	{load_complete, Pid, Mg, Error} ->
638	    i("await_multi_load_collectors -> "
639	      "received error complete from "
640	      "~n   ~p [~p]: "
641	      "~n   ~p", [Pid, Mg, Error]),
642	    Pids2 = lists:keydelete(Pid, 1, Pids),
643	    Errs2 = [{Mg, Error}|Errs],
644	    await_multi_load_collectors(Pids2, Oks, Errs2);
645
646	{'EXIT', Pid, normal} ->
647	    %% This is assumed to be one of the collectors
648	    i("await_multi_load_collectors -> "
649	      "received (normal) exit signal from ~p", [Pid]),
650	    await_multi_load_collectors(Pids, Oks, Errs);
651
652	{'EXIT', Pid, Reason} ->
653	    i("await_multi_load_collectors -> "
654	      "received unexpected exit from ~p:"
655	      "~n   ~p", [Pid, Reason]),
656	    case lists:keydelete(Pid, 1, Pids) of
657		Pids ->
658		    %% Not one of my procs, or a proc I have already
659		    %% received a complete from.
660		    await_multi_load_collectors(Pids, Oks, Errs);
661		Pids2 ->
662		    [{Pid,Mg}] = Pids -- Pids2,
663		    Errs2 = [{Mg, {unexpected_exit, Reason}}|Errs],
664		    await_multi_load_collectors(Pids, Oks, Errs2)
665	    end;
666
667	Else ->
668	    i("await_multi_load_collectors -> received unexpected message:"
669	      "~n~p", [Else]),
670	    await_multi_load_collectors(Pids, Oks, Errs)
671    after
672	5000 ->
673	    i("await_multi_load_collectors -> still awaiting reply from:"
674	      "~n~p", [Pids]),
675	    await_multi_load_collectors(Pids, Oks, Errs)
676    end.
677
678
679%% Note that this is an approximation...we run all the
680%% MGs in parrallel, so it should be "accurate"...
681multi_load_collector_calc(Sec, Oks) ->
682    Succs = lists:sum([Ok   || {_, Ok,   _} <- Oks]),
683    Fails = lists:sum([Err  || {_,  _, Err} <- Oks]),
684    io:format("~ntotal multiple loaders result: ~n"
685	      "   Number of successfull: ~w~n"
686	      "   Number of failure:     ~w~n"
687	      "   Total Calls / seconds: ~.2f~n~n",
688	      [Succs, Fails, Sec]),
689    ok.
690
691%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
692
693make_node_names(Name, Num) ->
694    make_node_names(Name, Num, []).
695
696make_node_names(_, 0, Names) ->
697    Names;
698make_node_names(BaseName, N, Names) ->
699    Name = lists:flatten(io_lib:format("~p~w", [BaseName,N])),
700    make_node_names(BaseName, N-1, [make_node_name(Name)|Names]).
701
702make_node_name(Name) when is_atom(Name) ->
703    make_node_name(atom_to_list(Name));
704make_node_name(Name) when is_list(Name) ->
705    case string:tokens(atom_to_list(node()), [$@]) of
706	[_,Host] ->
707	    list_to_atom(lists:concat([Name ++ "@" ++ Host]));
708	_ ->
709	    exit("Test node must be started with '-sname'")
710     end.
711
712make_mids(MgNodes) when is_list(MgNodes) andalso (length(MgNodes) > 0) ->
713    make_mids(MgNodes, []).
714
715make_mids([], Mids) ->
716    lists:reverse(Mids);
717make_mids([MgNode|MgNodes], Mids) ->
718    case string:tokens(atom_to_list(MgNode), [$@]) of
719	[Name, _] ->
720	    Mid = {deviceName, Name},
721	    make_mids(MgNodes, [{Mid, MgNode}|Mids]);
722	_Else ->
723	    exit("Test node must be started with '-sname'")
724    end.
725
726maybe_display_system_info(NumLoaders) when NumLoaders > 50 ->
727    [{display_system_info, timer:seconds(2)}];
728maybe_display_system_info(NumLoaders) when NumLoaders > 10 ->
729    [{display_system_info, timer:seconds(1)}];
730maybe_display_system_info(_) ->
731    [].
732
733
734%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
735
736try_tc(TCName, Pre, Case, Post) ->
737    try_tc(TCName, "TEST", ?TEST_VERBOSITY, Pre, Case, Post).
738
739try_tc(TCName, Name, Verbosity, Pre, Case, Post) ->
740    ?TRY_TC(TCName, Name, Verbosity, Pre, Case, Post).
741
742
743%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
744
745min(M) -> timer:minutes(M).
746
747p(F, A) ->
748    io:format("*** [~s] ~p ***"
749	      "~n   " ++ F ++ "~n",
750	      [?FTS(), self() | A]).
751
752i(F) ->
753    i(F, []).
754
755i(F, A) ->
756    print(info, get(verbosity), get(tc), "INF", F, A).
757
758d(F) ->
759    d(F, []).
760
761d(F, A) ->
762    print(debug, get(verbosity), get(tc), "DBG", F, A).
763
764printable(_, debug)   -> true;
765printable(info, info) -> true;
766printable(_,_)        -> false.
767
768print(Severity, Verbosity, Tc, P, F, A) ->
769    print(printable(Severity,Verbosity), Tc, P, F, A).
770
771print(true, Tc, P, F, A) ->
772    io:format("*** [~s] ~s ~p ~s:~w ***"
773              "~n   " ++ F ++ "~n",
774              [?FTS(), P, self(), get(sname), Tc | A]);
775print(_, _, _, _, _) ->
776    ok.
777
778
779