1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2003-2016. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20
21%%
22%%----------------------------------------------------------------------
23%% Purpose: Verify the application specifics of the Megaco application
24%%----------------------------------------------------------------------
25-module(megaco_load_test).
26
27-compile(export_all).
28
29-include("megaco_test_lib.hrl").
30-include_lib("megaco/include/megaco.hrl").
31-include_lib("megaco/include/megaco_message_v1.hrl").
32
33-define(TEST_VERBOSITY, debug).
34-define(MGC_VERBOSITY,  silence).
35-define(MG_VERBOSITY,   silence).
36
37-define(SINGLE_USER_LOAD_NUM_REQUESTS, 1000).
38-define(MULTI_USER_LOAD_NUM_REQUESTS,  1000).
39
40-define(MGC_START(Pid, Mid, ET, Conf, Verb),
41        megaco_test_mgc:start(Pid, Mid, ET,
42			      [{megaco_trace, false}] ++ Conf, Verb)).
43-define(MGC_STOP(Pid), megaco_test_mgc:stop(Pid)).
44-define(MGC_USER_INFO(Pid,Tag), megaco_test_mgc:user_info(Pid,Tag)).
45-define(MGC_CONN_INFO(Pid,Tag), megaco_test_mgc:conn_info(Pid,Tag)).
46-define(MGC_SET_VERBOSITY(Pid, V), megaco_test_mgc:verbosity(Pid, V)).
47
48-define(MG_START(Pid, Mid, Enc, Transp, Conf, Verb),
49        megaco_test_mg:start(Pid, Mid, Enc, Transp,
50			     [{megaco_trace, false},
51			      {transport_opts, [{serialize, true}]}] ++ Conf,
52			     Verb)).
53-define(MG_STOP(Pid), megaco_test_mg:stop(Pid)).
54-define(MG_USER_INFO(Pid,Tag), megaco_test_mg:user_info(Pid,Tag)).
55-define(MG_CONN_INFO(Pid,Tag), megaco_test_mg:conn_info(Pid,Tag)).
56-define(MG_SERV_CHANGE(Pid), megaco_test_mg:service_change(Pid)).
57-define(MG_MLOAD(Pid, NL, NR),
58	timer:tc(megaco_test_mg, apply_multi_load, [Pid, NL, NR])).
59-define(MG_LOAD(Pid, NL, NR), megaco_test_mg:apply_multi_load(Pid, NL, NR)).
60-define(MG_SET_VERBOSITY(Pid, V), megaco_test_mg:verbosity(Pid, V)).
61
62
63%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
64
65
66t()     -> megaco_test_lib:t(?MODULE).
67t(Case) -> megaco_test_lib:t({?MODULE, Case}).
68
69min(M) -> timer:minutes(M).
70
71%% Test server callbacks
72init_per_testcase(single_user_light_load = Case, Config) ->
73    C = lists:keydelete(tc_timeout, 1, Config),
74    do_init_per_testcase(Case, [{tc_timeout, min(2)}|C]);
75init_per_testcase(single_user_medium_load = Case, Config) ->
76    C = lists:keydelete(tc_timeout, 1, Config),
77    do_init_per_testcase(Case, [{tc_timeout, min(5)}|C]);
78init_per_testcase(single_user_heavy_load = Case, Config) ->
79    C = lists:keydelete(tc_timeout, 1, Config),
80    do_init_per_testcase(Case, [{tc_timeout, min(10)}|C]);
81init_per_testcase(single_user_extreme_load = Case, Config) ->
82    C = lists:keydelete(tc_timeout, 1, Config),
83    do_init_per_testcase(Case, [{tc_timeout, min(20)}|C]);
84init_per_testcase(multi_user_light_load = Case, Config) ->
85    C = lists:keydelete(tc_timeout, 1, Config),
86    do_init_per_testcase(Case, [{tc_timeout, min(2)}|C]);
87init_per_testcase(multi_user_medium_load = Case, Config) ->
88    C = lists:keydelete(tc_timeout, 1, Config),
89    do_init_per_testcase(Case, [{tc_timeout, min(5)}|C]);
90init_per_testcase(multi_user_heavy_load = Case, Config) ->
91    C = lists:keydelete(tc_timeout, 1, Config),
92    do_init_per_testcase(Case, [{tc_timeout, min(10)}|C]);
93init_per_testcase(multi_user_extreme_load = Case, Config) ->
94    C = lists:keydelete(tc_timeout, 1, Config),
95    do_init_per_testcase(Case, [{tc_timeout, min(20)}|C]);
96init_per_testcase(Case, Config) ->
97    do_init_per_testcase(Case, Config).
98
99do_init_per_testcase(Case, Config) ->
100    process_flag(trap_exit, true),
101    megaco_test_lib:init_per_testcase(Case, Config).
102
103end_per_testcase(Case, Config) ->
104    process_flag(trap_exit, false),
105    megaco_test_lib:end_per_testcase(Case, Config).
106
107
108%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
109
110all() ->
111    [single_user_light_load,
112     single_user_medium_load, single_user_heavy_load,
113     single_user_extreme_load, multi_user_light_load,
114     multi_user_medium_load, multi_user_heavy_load,
115     multi_user_extreme_load].
116
117groups() ->
118    [].
119
120init_per_group(_GroupName, Config) ->
121    Config.
122
123end_per_group(_GroupName, Config) ->
124    Config.
125
126
127
128%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
129
130single_user_light_load(suite) ->
131    [];
132single_user_light_load(doc) ->
133    [];
134single_user_light_load(Config) when is_list(Config) ->
135    put(verbosity, ?TEST_VERBOSITY),
136    put(tc,        single_user_light_load),
137    put(sname,     "TEST"),
138    i("starting"),
139
140    load_controller(Config,
141		    fun(Env) ->
142			    populate(Env),
143			    exit( single_user_load(5) )
144		    end),
145
146    i("done", []),
147    ok.
148
149
150
151%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
152
153single_user_medium_load(suite) ->
154    [];
155single_user_medium_load(doc) ->
156    [];
157single_user_medium_load(Config) when is_list(Config) ->
158    put(verbosity, ?TEST_VERBOSITY),
159    put(tc,        single_user_medium_load),
160    put(sname,     "TEST"),
161    i("starting"),
162
163    load_controller(Config,
164		    fun(Env) ->
165			    populate(Env),
166			    exit( single_user_load(15) )
167		    end),
168
169    i("done", []),
170    ok.
171
172
173%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
174
175single_user_heavy_load(suite) ->
176    [];
177single_user_heavy_load(doc) ->
178    [];
179single_user_heavy_load(Config) when is_list(Config) ->
180    put(verbosity, ?TEST_VERBOSITY),
181    put(tc,        single_user_heavy_load),
182    put(sname,     "TEST"),
183    i("starting"),
184
185    load_controller(Config,
186		    fun(Env) ->
187			    populate(Env),
188			    exit( single_user_load(25) )
189		    end),
190
191    i("done", []),
192    ok.
193
194
195%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
196
197single_user_extreme_load(suite) ->
198    [];
199single_user_extreme_load(doc) ->
200    [];
201single_user_extreme_load(Config) when is_list(Config) ->
202    put(verbosity, ?TEST_VERBOSITY),
203    put(tc,        single_user_extreme_load),
204    put(sname,     "TEST"),
205    i("starting"),
206
207    load_controller(Config,
208		    fun(Env) ->
209			    populate(Env),
210			    exit( single_user_load(100) )
211		    end),
212
213    i("done", []),
214    ok.
215
216
217%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
218
219multi_user_light_load(suite) ->
220    [];
221multi_user_light_load(doc) ->
222    [];
223multi_user_light_load(Config) when is_list(Config) ->
224    put(verbosity, ?TEST_VERBOSITY),
225    put(tc,        multi_user_light_load),
226    put(sname,     "TEST"),
227    i("starting"),
228
229    load_controller(Config,
230		    fun(Env) ->
231			    populate(Env),
232			    exit( multi_user_load(3,1) )
233		    end),
234
235    i("done", []),
236    ok.
237
238
239%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
240
241multi_user_medium_load(suite) ->
242    [];
243multi_user_medium_load(doc) ->
244    [];
245multi_user_medium_load(Config) when is_list(Config) ->
246    put(verbosity, ?TEST_VERBOSITY),
247    put(tc,        multi_user_medium_load),
248    put(sname,     "TEST"),
249    i("starting"),
250
251    load_controller(Config,
252		    fun(Env) ->
253			    populate(Env),
254			    exit( multi_user_load(3,5) )
255		    end),
256
257    i("done", []),
258    ok.
259
260
261%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
262
263multi_user_heavy_load(suite) ->
264    [];
265multi_user_heavy_load(doc) ->
266    [];
267multi_user_heavy_load(Config) when is_list(Config) ->
268    put(verbosity, ?TEST_VERBOSITY),
269    put(tc,        multi_user_heavy_load),
270    put(sname,     "TEST"),
271    i("starting"),
272
273    load_controller(Config,
274		    fun(Env) ->
275			    populate(Env),
276			    exit( multi_user_load(3,10) )
277		    end),
278
279    i("done", []),
280    ok.
281
282
283%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
284
285multi_user_extreme_load(suite) ->
286    [];
287multi_user_extreme_load(doc) ->
288    [];
289multi_user_extreme_load(Config) when is_list(Config) ->
290    put(verbosity, ?TEST_VERBOSITY),
291    put(tc,        multi_user_extreme_load),
292    put(sname,     "TEST"),
293    i("starting"),
294
295    load_controller(Config,
296		    fun(Env) ->
297			    populate(Env),
298			    exit( multi_user_load(3,15) )
299		    end),
300
301    i("done", []),
302    ok.
303
304
305%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
306
307populate([]) ->
308    ok;
309populate([{Key,Val}|Env]) ->
310    put(Key, Val),
311    populate(Env).
312
313load_controller(Config, Fun) when is_list(Config) and is_function(Fun) ->
314    process_flag(trap_exit, true),
315    {value, {tc_timeout, TcTimeout}} =
316	lists:keysearch(tc_timeout, 1, Config),
317    SkipTimeout = trunc(95*TcTimeout/100), % 95% of TcTimeout
318    Env = get(),
319    Loader = erlang:spawn_link(fun() -> Fun(Env) end),
320    receive
321	{'EXIT', Loader, normal} ->
322	    d("load_controller -> "
323	      "loader [~p] terminated with normal", [Loader]),
324	    ok;
325	{'EXIT', Loader, ok} ->
326	    d("load_controller -> "
327	      "loader [~p] terminated with ok~n", [Loader]),
328	    ok;
329	{'EXIT', Loader, Reason} ->
330	    i("load_controller -> "
331	      "loader [~p] terminated with"
332	      "~n   ~p", [Loader, Reason]),
333	    erlang:error({unexpected_loader_result, Reason})
334    after SkipTimeout ->
335	    i("load_controller -> "
336	      "loader [~p] timeout", [Loader]),
337	    exit(Loader, kill),
338	    ?SKIP({timeout, SkipTimeout, TcTimeout})
339    end.
340
341%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
342
343single_user_load(NumLoaders) ->
344    MgcNode = make_node_name(mgc),
345    MgNode  = make_node_name(mg),
346    d("Nodes: "
347      "~n   MgcNode: ~p"
348      "~n   MgNode:  ~p", [MgcNode, MgNode]),
349    ok = megaco_test_lib:start_nodes([MgcNode, MgNode], ?FILE, ?LINE),
350
351    %% Start the MGC and MGs
352    i("[MGC] start"),
353    MgcMid = {deviceName, "ctrl"},
354    ET     = [{text, tcp, [{serialize, true}]}],
355    DSI = maybe_display_system_info(NumLoaders),
356    {ok, Mgc} = ?MGC_START(MgcNode, MgcMid, ET, DSI, ?MGC_VERBOSITY),
357
358    i("[MG] start"),
359    MgMid = {deviceName, "mg"},
360    {ok, Mg} = ?MG_START(MgNode, MgMid, text, tcp, DSI, ?MG_VERBOSITY),
361
362    d("MG user info: ~p", [?MG_USER_INFO(Mg, all)]),
363
364    i("[MG] connect to the MGC (service change)"),
365    ServChRes = ?MG_SERV_CHANGE(Mg),
366    d("service change result: ~p", [ServChRes]),
367
368    megaco_test_mg:update_conn_info(Mg,reply_timer,1000),
369    megaco_test_mgc:update_conn_info(Mgc,reply_timer,1000),
370
371    d("MG conn info: ~p", [?MG_CONN_INFO(Mg, all)]),
372
373    d("apply the load"),
374    Res = ?MG_MLOAD(Mg, NumLoaders, ?SINGLE_USER_LOAD_NUM_REQUESTS),
375    case Res of
376	{Time, {ok, Ok, Err}} ->
377	    Sec = Time / 1000000,
378	    io:format("~nmultiple loaders result: ~n"
379		      "   Number of successfull: ~w~n"
380		      "   Number of failure:     ~w~n"
381		      "   Time:                  ~w seconds~n"
382		      "   Calls / seconds        ~w~n~n",
383		      [Ok, Err, Sec, (NumLoaders * ?SINGLE_USER_LOAD_NUM_REQUESTS)/Sec]);
384	{Time, Error} ->
385	    io:format("SUL: multiple loaders failed: ~p after ~w~n",
386		      [Error, Time])
387    end,
388
389    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
390
391    i("verbosity to trace"),
392    ?MGC_SET_VERBOSITY(Mgc, debug),
393    ?MG_SET_VERBOSITY(Mg, debug),
394
395    %% Tell MG to stop
396    i("[MG] stop"),
397    ?MG_STOP(Mg),
398
399    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
400
401    %% Tell Mgc to stop
402    i("[MGC] stop"),
403    ?MGC_STOP(Mgc),
404
405    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
406
407    ok.
408
409
410
411%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
412
413multi_user_load(NumUsers, NumLoaders)
414  when (is_integer(NumUsers) andalso (NumUsers > 1) andalso
415	is_integer(NumLoaders) andalso (NumLoaders >= 1)) ->
416    MgcNode = make_node_name(mgc),
417    MgNodes = make_node_names(mg, NumUsers),
418    d("Nodes: "
419      "~n   MgcNode: ~p"
420      "~n   MgNodes: ~p", [MgcNode, MgNodes]),
421    ok = megaco_test_lib:start_nodes([MgcNode| MgNodes], ?FILE, ?LINE),
422
423    %% Start the MGC and MGs
424    i("[MGC] start"),
425    MgcMid = {deviceName, "ctrl"},
426    ET     = [{text, tcp, [{serialize, false}]}],
427    DSI = maybe_display_system_info(2 * NumUsers * NumLoaders),
428    {ok, Mgc} = ?MGC_START(MgcNode, MgcMid, ET, DSI, ?MGC_VERBOSITY),
429
430    megaco_test_mgc:update_user_info(Mgc,reply_timer,1000),
431    d("MGC user info: ~p", [?MGC_USER_INFO(Mgc, all)]),
432
433    MgUsers = make_mids(MgNodes),
434
435    d("start MGs, apply the load and stop MGs"),
436    ok = multi_load(MgUsers, DSI, NumLoaders, ?MULTI_USER_LOAD_NUM_REQUESTS),
437
438    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
439
440    ?MGC_SET_VERBOSITY(Mgc, debug),
441
442    %% Tell Mgc to stop
443    i("[MGC] stop"),
444    ?MGC_STOP(Mgc),
445
446    i("flush the message queue: ~p", [megaco_test_lib:flush()]),
447
448    ok.
449
450
451multi_load(MGs, DSI, NumLoaders, NumReqs) ->
452    d("multi_load -> entry with"
453      "~n   MGs:        ~p"
454      "~n   DSI:        ~p"
455      "~n   NumLoaders: ~p"
456      "~n   NumReqs:    ~p", [MGs, DSI, NumLoaders, NumReqs]),
457
458    Pids = multi_load_collector_start(MGs, DSI, NumLoaders, NumReqs, []),
459    case timer:tc(?MODULE, do_multi_load, [Pids, NumLoaders, NumReqs]) of
460	{Time, {ok, OKs, []}} ->
461	    Sec = Time / 1000000,
462	    multi_load_collector_calc(Sec, OKs);
463	{Time, Error} ->
464	    Sec = Time/1000000,
465	    io:format("~nmulti load failed after ~.1f:~n~p~n~n", [Sec,Error]),
466	    {error, Error}
467    end.
468
469do_multi_load(Pids, _NumLoaders, _NumReqs) ->
470    Fun =
471	fun({P,_}) ->
472		d("apply multi load for ~p", [P]),
473		P ! {apply_multi_load, self()}
474	end,
475    lists:foreach(Fun, Pids),
476    await_multi_load_collectors(Pids, [], []).
477
478multi_load_collector_start([], _DSI, _NumLoaders, _NumReqs, Pids) ->
479    Pids;
480multi_load_collector_start([{Mid, Node}|MGs], DSI, NumLoaders, NumReqs, Pids) ->
481    Env = get(),
482    Pid = spawn_link(?MODULE, multi_load_collector,
483		     [self(), Node, Mid, DSI, NumLoaders, NumReqs, Env]),
484    multi_load_collector_start(MGs, DSI, NumLoaders, NumReqs, [{Pid,Mid}|Pids]).
485
486get_env(Key, Env) ->
487    case lists:keysearch(Key, 1, Env) of
488	{value, {Key, Val}} ->
489	    Val;
490	_ ->
491	    undefined
492    end.
493
494multi_load_collector(Parent, Node, Mid, DSI, NumLoaders, NumReqs, Env) ->
495    put(verbosity, get_env(verbosity, Env)),
496    put(tc, get_env(tc, Env)),
497    put(sname, get_env(sname, Env) ++ "-loader"),
498    case ?MG_START(Node, Mid, text, tcp, DSI, ?MG_VERBOSITY) of
499	{ok, Pid} ->
500	    d("MG ~p user info: ~n~p", [Mid, ?MG_USER_INFO(Pid,all)]),
501	    ServChRes = ?MG_SERV_CHANGE(Pid),
502	    d("service change result: ~p", [ServChRes]),
503	    megaco_test_mg:update_conn_info(Pid,reply_timer,1000),
504	    d("MG ~p conn info: ~p", [Mid, ?MG_CONN_INFO(Pid,all)]),
505	    multi_load_collector_loop(Parent, Pid, Mid, NumLoaders, NumReqs);
506	Else ->
507	    Parent ! {load_start_failed, self(), Mid, Else}
508    end.
509
510multi_load_collector_loop(Parent, Pid, Mid, NumLoaders, NumReqs) ->
511    d("multi_load_collector_loop -> entry with"
512      "~n   Parent:     ~p"
513      "~n   Pid:        ~p"
514      "~n   Mid:        ~p"
515      "~n   NumLoaders: ~p"
516      "~n   NumReqs:    ~p"
517      "~nwhen"
518      "~n   self():     ~p"
519      "~n   node():     ~p",
520      [Parent, Pid, Mid, NumLoaders, NumReqs, self(), node()]),
521    receive
522	{apply_multi_load, Parent} ->
523	    Res = ?MG_LOAD(Pid, NumLoaders, NumReqs),
524	    Parent ! {load_complete, self(), Mid, Res},
525	    ?MG_SET_VERBOSITY(Pid, debug),
526	    ?MG_STOP(Pid),
527	    exit(normal)
528    end.
529
530
531await_multi_load_collectors([], Oks, Errs) ->
532    i("await_multi_load_collectors -> done"),
533    {ok, Oks, Errs};
534await_multi_load_collectors(Pids, Oks, Errs) ->
535    receive
536	{load_complete, Pid, Mg, {ok, Ok, Err}} ->
537	    i("await_multi_load_collectors -> "
538	      "received ok complete from "
539	      "~n   ~p [~p]", [Pid, Mg]),
540	    Pids2 = lists:keydelete(Pid, 1, Pids),
541	    Oks2  = [{Mg, Ok, Err}|Oks],
542	    await_multi_load_collectors(Pids2, Oks2, Errs);
543	{load_complete, Pid, Mg, Error} ->
544	    i("await_multi_load_collectors -> "
545	      "received error complete from "
546	      "~n   ~p [~p]: "
547	      "~n   ~p", [Pid, Mg, Error]),
548	    Pids2 = lists:keydelete(Pid, 1, Pids),
549	    Errs2 = [{Mg, Error}|Errs],
550	    await_multi_load_collectors(Pids2, Oks, Errs2);
551
552	{'EXIT', Pid, normal} ->
553	    %% This is assumed to be one of the collectors
554	    i("await_multi_load_collectors -> "
555	      "received (normal) exit signal from ~p", [Pid]),
556	    await_multi_load_collectors(Pids, Oks, Errs);
557
558	{'EXIT', Pid, Reason} ->
559	    i("await_multi_load_collectors -> "
560	      "received unexpected exit from ~p:"
561	      "~n   ~p", [Pid, Reason]),
562	    case lists:keydelete(Pid, 1, Pids) of
563		Pids ->
564		    %% Not one of my procs, or a proc I have already
565		    %% received a complete from.
566		    await_multi_load_collectors(Pids, Oks, Errs);
567		Pids2 ->
568		    [{Pid,Mg}] = Pids -- Pids2,
569		    Errs2 = [{Mg, {unexpected_exit, Reason}}|Errs],
570		    await_multi_load_collectors(Pids, Oks, Errs2)
571	    end;
572
573	Else ->
574	    i("await_multi_load_collectors -> received unexpected message:"
575	      "~n~p", [Else]),
576	    await_multi_load_collectors(Pids, Oks, Errs)
577    after
578	5000 ->
579	    i("await_multi_load_collectors -> still awaiting reply from:"
580	      "~n~p", [Pids]),
581	    await_multi_load_collectors(Pids, Oks, Errs)
582    end.
583
584
585%% Note that this is an approximation...we run all the
586%% MGs in parrallel, so it should be "accurate"...
587multi_load_collector_calc(Sec, Oks) ->
588    Succs = lists:sum([Ok   || {_, Ok,   _} <- Oks]),
589    Fails = lists:sum([Err  || {_,  _, Err} <- Oks]),
590    io:format("~ntotal multiple loaders result: ~n"
591	      "   Number of successfull: ~w~n"
592	      "   Number of failure:     ~w~n"
593	      "   Total Calls / seconds: ~.2f~n~n",
594	      [Succs, Fails, Sec]),
595    ok.
596
597%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
598
599make_node_names(Name, Num) ->
600    make_node_names(Name, Num, []).
601
602make_node_names(_, 0, Names) ->
603    Names;
604make_node_names(BaseName, N, Names) ->
605    Name = lists:flatten(io_lib:format("~p~w", [BaseName,N])),
606    make_node_names(BaseName, N-1, [make_node_name(Name)|Names]).
607
608make_node_name(Name) when is_atom(Name) ->
609    make_node_name(atom_to_list(Name));
610make_node_name(Name) when is_list(Name) ->
611    case string:tokens(atom_to_list(node()), [$@]) of
612	[_,Host] ->
613	    list_to_atom(lists:concat([Name ++ "@" ++ Host]));
614	_ ->
615	    exit("Test node must be started with '-sname'")
616     end.
617
618make_mids(MgNodes) when is_list(MgNodes) andalso (length(MgNodes) > 0) ->
619    make_mids(MgNodes, []).
620
621make_mids([], Mids) ->
622    lists:reverse(Mids);
623make_mids([MgNode|MgNodes], Mids) ->
624    case string:tokens(atom_to_list(MgNode), [$@]) of
625	[Name, _] ->
626	    Mid = {deviceName, Name},
627	    make_mids(MgNodes, [{Mid, MgNode}|Mids]);
628	_Else ->
629	    exit("Test node must be started with '-sname'")
630    end.
631
632tim() ->
633    {A,B,C} = erlang:now(),
634    A*1000000000+B*1000+(C div 1000).
635
636sleep(X) -> receive after X -> ok end.
637
638error_msg(F,A) -> error_logger:error_msg(F ++ "~n",A).
639
640maybe_display_system_info(NumLoaders) when NumLoaders > 50 ->
641    [{display_system_info, timer:seconds(2)}];
642maybe_display_system_info(NumLoaders) when NumLoaders > 10 ->
643    [{display_system_info, timer:seconds(1)}];
644maybe_display_system_info(_) ->
645    [].
646
647
648%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
649
650i(F) ->
651    i(F, []).
652
653i(F, A) ->
654    print(info, get(verbosity), now(), get(tc), "INF", F, A).
655
656d(F) ->
657    d(F, []).
658
659d(F, A) ->
660    print(debug, get(verbosity), now(), get(tc), "DBG", F, A).
661
662printable(_, debug)   -> true;
663printable(info, info) -> true;
664printable(_,_)        -> false.
665
666print(Severity, Verbosity, Ts, Tc, P, F, A) ->
667    print(printable(Severity,Verbosity), Ts, Tc, P, F, A).
668
669print(true, Ts, Tc, P, F, A) ->
670    io:format("*** [~s] ~s ~p ~s:~w ***"
671              "~n   " ++ F ++ "~n",
672              [format_timestamp(Ts), P, self(), get(sname), Tc | A]);
673print(_, _, _, _, _, _) ->
674    ok.
675
676
677%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
678
679random_init() ->
680    {A,B,C} = now(),
681    random:seed(A,B,C).
682
683random() ->
684    10 * random:uniform(50).
685
686apply_load_timer() ->
687    erlang:send_after(random(), self(), apply_load_timeout).
688
689format_timestamp({_N1, _N2, N3} = Now) ->
690    {Date, Time}   = calendar:now_to_datetime(Now),
691    {YYYY,MM,DD}   = Date,
692    {Hour,Min,Sec} = Time,
693    FormatDate =
694        io_lib:format("~.4w:~.2.0w:~.2.0w ~.2.0w:~.2.0w:~.2.0w 4~w",
695                      [YYYY,MM,DD,Hour,Min,Sec,round(N3/1000)]),
696    lists:flatten(FormatDate).
697