%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2002-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
-module(test_server_ctrl).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% %%
%% The Erlang Test Server %%
%% %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%
%% MODULE DEPENDENCIES:
%% HARD TO REMOVE: erlang, lists, io_lib, gen_server, file, io, string,
%% code, ets, rpc, gen_tcp, inet, erl_tar, sets,
%% test_server, test_server_sup, test_server_node
%% EASIER TO REMOVE: filename, filelib, lib, re
%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% SUPERVISOR INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-export([start/0, start/1, start_link/1, stop/0]).
%%% OPERATOR INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-export([add_spec/1, add_dir/2, add_dir/3]).
-export([add_module/1, add_module/2,
add_conf/3,
add_case/2, add_case/3, add_cases/2, add_cases/3]).
-export([add_dir_with_skip/3, add_dir_with_skip/4, add_tests_with_skip/3]).
-export([add_module_with_skip/2, add_module_with_skip/3,
add_conf_with_skip/4,
add_case_with_skip/3, add_case_with_skip/4,
add_cases_with_skip/3, add_cases_with_skip/4]).
-export([jobs/0, run_test/1, wait_finish/0, idle_notify/1,
abort_current_testcase/1, abort/0]).
-export([start_get_totals/1, stop_get_totals/0]).
-export([reject_io_reqs/1, get_levels/0, set_levels/3]).
-export([multiply_timetraps/1, scale_timetraps/1, get_timetrap_parameters/0]).
-export([create_priv_dir/1]).
-export([cover/1, cover/2, cover/3,
cover_compile/7, cover_analyse/2, cross_cover_analyse/2,
trc/1, stop_trace/0]).
-export([testcase_callback/1]).
-export([set_random_seed/1]).
-export([kill_slavenodes/0]).
%%% TEST_SERVER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-export([print/2, print/3, print/4, print_timestamp/2]).
-export([start_node/3, stop_node/1, wait_for_node/1, is_release_available/1]).
-export([format/1, format/2, format/3, to_string/1]).
-export([get_target_info/0]).
-export([get_hosts/0]).
-export([node_started/1]).
-export([uri_encode/1,uri_encode/2]).
%%% DEBUGGER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-export([i/0, p/1, p/3, pi/2, pi/4, t/0, t/1]).
%%% PRIVATE EXPORTED %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-export([init/1, terminate/2]).
-export([handle_call/3, handle_cast/2, handle_info/2]).
-export([do_test_cases/4]).
-export([do_spec/2, do_spec_list/2]).
-export([xhtml/2, escape_chars/1]).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-include("test_server_internal.hrl").
-include_lib("kernel/include/file.hrl").
-define(suite_ext, "_SUITE").
-define(log_ext, ".log.html").
-define(src_listing_ext, ".src.html").
-define(logdir_ext, ".logs").
-define(data_dir_suffix, "_data/").
-define(suitelog_name, "suite.log").
-define(suitelog_latest_name, "suite.log.latest").
-define(coverlog_name, "cover.html").
-define(raw_coverlog_name, "cover.log").
-define(cross_coverlog_name, "cross_cover.html").
-define(raw_cross_coverlog_name, "cross_cover.log").
-define(cross_cover_info, "cross_cover.info").
-define(cover_total, "total_cover.log").
-define(unexpected_io_log, "unexpected_io.log.html").
-define(last_file, "last_name").
-define(last_link, "last_link").
-define(last_test, "last_test").
-define(html_ext, ".html").
-define(now, os:timestamp()).
-define(void_fun, fun() -> ok end).
-define(mod_result(X), if X == skip -> skipped;
X == auto_skip -> skipped;
true -> X end).
-define(auto_skip_color, "#FFA64D").
-define(user_skip_color, "#FF8000").
-define(sortable_table_name, "SortableTable").
-record(state,{jobs=[], levels={1,19,10}, reject_io_reqs=false,
multiply_timetraps=1, scale_timetraps=true,
create_priv_dir=auto_per_run, finish=false,
target_info, trc=false, cover=false, wait_for_node=[],
testcase_callback=undefined, idle_notify=[],
get_totals=false, random_seed=undefined}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% OPERATOR INTERFACE
add_dir(Name, Job=[Dir|_Dirs]) when is_list(Dir) ->
add_job(cast_to_list(Name),
lists:map(fun(D)-> {dir,cast_to_list(D)} end, Job));
add_dir(Name, Dir) ->
add_job(cast_to_list(Name), {dir,cast_to_list(Dir)}).
add_dir(Name, Job=[Dir|_Dirs], Pattern) when is_list(Dir) ->
add_job(cast_to_list(Name),
lists:map(fun(D)-> {dir,cast_to_list(D),
cast_to_list(Pattern)} end, Job));
add_dir(Name, Dir, Pattern) ->
add_job(cast_to_list(Name), {dir,cast_to_list(Dir),cast_to_list(Pattern)}).
add_module(Mod) when is_atom(Mod) ->
add_job(atom_to_list(Mod), {Mod,all}).
add_module(Name, Mods) when is_list(Mods) ->
add_job(cast_to_list(Name), lists:map(fun(Mod) -> {Mod,all} end, Mods)).
add_conf(Name, Mod, Conf) when is_tuple(Conf) ->
add_job(cast_to_list(Name), {Mod,[Conf]});
add_conf(Name, Mod, Confs) when is_list(Confs) ->
add_job(cast_to_list(Name), {Mod,Confs}).
add_case(Mod, Case) when is_atom(Mod), is_atom(Case) ->
add_job(atom_to_list(Mod), {Mod,Case}).
add_case(Name, Mod, Case) when is_atom(Mod), is_atom(Case) ->
add_job(Name, {Mod,Case}).
add_cases(Mod, Cases) when is_atom(Mod), is_list(Cases) ->
add_job(atom_to_list(Mod), {Mod,Cases}).
add_cases(Name, Mod, Cases) when is_atom(Mod), is_list(Cases) ->
add_job(Name, {Mod,Cases}).
add_spec(Spec) ->
Name = filename:rootname(Spec, ".spec"),
case filelib:is_file(Spec) of
true -> add_job(Name, {spec,Spec});
false -> {error,nofile}
end.
%% This version of the interface is to be used if there are
%% suites or cases that should be skipped.
add_dir_with_skip(Name, Job=[Dir|_Dirs], Skip) when is_list(Dir) ->
add_job(cast_to_list(Name),
lists:map(fun(D)-> {dir,cast_to_list(D)} end, Job),
Skip);
add_dir_with_skip(Name, Dir, Skip) ->
add_job(cast_to_list(Name), {dir,cast_to_list(Dir)}, Skip).
add_dir_with_skip(Name, Job=[Dir|_Dirs], Pattern, Skip) when is_list(Dir) ->
add_job(cast_to_list(Name),
lists:map(fun(D)-> {dir,cast_to_list(D),
cast_to_list(Pattern)} end, Job),
Skip);
add_dir_with_skip(Name, Dir, Pattern, Skip) ->
add_job(cast_to_list(Name),
{dir,cast_to_list(Dir),cast_to_list(Pattern)}, Skip).
add_module_with_skip(Mod, Skip) when is_atom(Mod) ->
add_job(atom_to_list(Mod), {Mod,all}, Skip).
add_module_with_skip(Name, Mods, Skip) when is_list(Mods) ->
add_job(cast_to_list(Name), lists:map(fun(Mod) -> {Mod,all} end, Mods), Skip).
add_conf_with_skip(Name, Mod, Conf, Skip) when is_tuple(Conf) ->
add_job(cast_to_list(Name), {Mod,[Conf]}, Skip);
add_conf_with_skip(Name, Mod, Confs, Skip) when is_list(Confs) ->
add_job(cast_to_list(Name), {Mod,Confs}, Skip).
add_case_with_skip(Mod, Case, Skip) when is_atom(Mod), is_atom(Case) ->
add_job(atom_to_list(Mod), {Mod,Case}, Skip).
add_case_with_skip(Name, Mod, Case, Skip) when is_atom(Mod), is_atom(Case) ->
add_job(Name, {Mod,Case}, Skip).
add_cases_with_skip(Mod, Cases, Skip) when is_atom(Mod), is_list(Cases) ->
add_job(atom_to_list(Mod), {Mod,Cases}, Skip).
add_cases_with_skip(Name, Mod, Cases, Skip) when is_atom(Mod), is_list(Cases) ->
add_job(Name, {Mod,Cases}, Skip).
add_tests_with_skip(LogDir, Tests, Skip) ->
add_job(LogDir,
lists:map(fun({Dir,all,all}) ->
{Dir,{dir,Dir}};
({Dir,Mods,all}) ->
{Dir,lists:map(fun(M) -> {M,all} end, Mods)};
({Dir,Mod,Cases}) ->
{Dir,{Mod,Cases}}
end, Tests),
Skip).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% COMMAND LINE INTERFACE
parse_cmd_line(Cmds) ->
parse_cmd_line(Cmds, [], [], local, false, false, undefined).
parse_cmd_line(['SPEC',Spec|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
case file:consult(Spec) of
{ok, TermList} ->
Name = filename:rootname(Spec),
parse_cmd_line(Cmds, TermList++SpecList, [Name|Names], Param,
Trc, Cov, TCCB);
{error,Reason} ->
io:format("Can't open ~tw: ~tp\n",[Spec, file:format_error(Reason)]),
parse_cmd_line(Cmds, SpecList, Names, Param, Trc, Cov, TCCB)
end;
parse_cmd_line(['NAME',Name|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
parse_cmd_line(Cmds, SpecList, [{name,atom_to_list(Name)}|Names],
Param, Trc, Cov, TCCB);
parse_cmd_line(['SKIPMOD',Mod|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
parse_cmd_line(Cmds, [{skip,{Mod,"by command line"}}|SpecList], Names,
Param, Trc, Cov, TCCB);
parse_cmd_line(['SKIPCASE',Mod,Case|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
parse_cmd_line(Cmds, [{skip,{Mod,Case,"by command line"}}|SpecList], Names,
Param, Trc, Cov, TCCB);
parse_cmd_line(['DIR',Dir|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
Name = filename:basename(Dir),
parse_cmd_line(Cmds, [{topcase,{dir,Name}}|SpecList], [Name|Names],
Param, Trc, Cov, TCCB);
parse_cmd_line(['MODULE',Mod|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
parse_cmd_line(Cmds,[{topcase,{Mod,all}}|SpecList],[atom_to_list(Mod)|Names],
Param, Trc, Cov, TCCB);
parse_cmd_line(['CASE',Mod,Case|Cmds], SpecList, Names, Param, Trc, Cov, TCCB) ->
parse_cmd_line(Cmds,[{topcase,{Mod,Case}}|SpecList],[atom_to_list(Mod)|Names],
Param, Trc, Cov, TCCB);
parse_cmd_line(['TRACE',Trc|Cmds], SpecList, Names, Param, _Trc, Cov, TCCB) ->
parse_cmd_line(Cmds, SpecList, Names, Param, Trc, Cov, TCCB);
parse_cmd_line(['COVER',App,CF,Analyse|Cmds], SpecList, Names, Param, Trc, _Cov, TCCB) ->
parse_cmd_line(Cmds, SpecList, Names, Param, Trc, {{App,CF}, Analyse}, TCCB);
parse_cmd_line(['TESTCASE_CALLBACK',Mod,Func|Cmds], SpecList, Names, Param, Trc, Cov, _) ->
parse_cmd_line(Cmds, SpecList, Names, Param, Trc, Cov, {Mod,Func});
parse_cmd_line([Obj|_Cmds], _SpecList, _Names, _Param, _Trc, _Cov, _TCCB) ->
io:format("~w: Bad argument: ~tw\n", [?MODULE,Obj]),
io:format(" Use the `ts' module to start tests.\n", []),
io:format(" (If you ARE using `ts', there is a bug in `ts'.)\n", []),
halt(1);
parse_cmd_line([], SpecList, Names, Param, Trc, Cov, TCCB) ->
NameList = lists:reverse(Names, ["suite"]),
Name = case lists:keysearch(name, 1, NameList) of
{value,{name,N}} -> N;
false -> hd(NameList)
end,
{lists:reverse(SpecList), Name, Param, Trc, Cov, TCCB}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% cast_to_list(X) -> string()
%% X = list() | atom() | void()
%% Returns a string representation of whatever was input
cast_to_list(X) when is_list(X) -> X;
cast_to_list(X) when is_atom(X) -> atom_to_list(X);
cast_to_list(X) -> lists:flatten(io_lib:format("~tw", [X])).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% START INTERFACE
%% Kept for backwards compatibility
start(_) ->
start().
start_link(_) ->
start_link().
start() ->
case gen_server:start({local,?MODULE}, ?MODULE, [], []) of
{error, {already_started, Pid}} ->
{ok, Pid};
Other ->
Other
end.
start_link() ->
case gen_server:start_link({local,?MODULE}, ?MODULE, [], []) of
{error, {already_started, Pid}} ->
{ok, Pid};
Other ->
Other
end.
run_test(CommandLine) ->
process_flag(trap_exit,true),
{SpecList,Name,Param,Trc,Cov,TCCB} = parse_cmd_line(CommandLine),
{ok,_TSPid} = start_link(Param),
case Trc of
false -> ok;
File -> trc(File)
end,
case Cov of
false -> ok;
{{App,CoverFile},Analyse} -> cover(App, maybe_file(CoverFile), Analyse)
end,
testcase_callback(TCCB),
add_job(Name, {command_line,SpecList}),
wait_finish().
%% Converted CoverFile to a string unless it is 'none'
maybe_file(none) ->
none;
maybe_file(CoverFile) ->
atom_to_list(CoverFile).
idle_notify(Fun) ->
{ok, Pid} = controller_call({idle_notify,Fun}),
Pid.
start_get_totals(Fun) ->
{ok, Pid} = controller_call({start_get_totals,Fun}),
Pid.
stop_get_totals() ->
ok = controller_call(stop_get_totals),
ok.
wait_finish() ->
OldTrap = process_flag(trap_exit, true),
{ok, Pid} = finish(true),
link(Pid),
receive
{'EXIT',Pid,_} ->
ok
end,
process_flag(trap_exit, OldTrap),
ok.
abort_current_testcase(Reason) ->
controller_call({abort_current_testcase,Reason}).
abort() ->
OldTrap = process_flag(trap_exit, true),
{ok, Pid} = finish(abort),
link(Pid),
receive
{'EXIT',Pid,_} ->
ok
end,
process_flag(trap_exit, OldTrap),
ok.
finish(Abort) ->
controller_call({finish,Abort}).
stop() ->
controller_call(stop).
jobs() ->
controller_call(jobs).
get_levels() ->
controller_call(get_levels).
set_levels(Show, Major, Minor) ->
controller_call({set_levels,Show,Major,Minor}).
reject_io_reqs(Bool) ->
controller_call({reject_io_reqs,Bool}).
multiply_timetraps(N) ->
controller_call({multiply_timetraps,N}).
scale_timetraps(Bool) ->
controller_call({scale_timetraps,Bool}).
get_timetrap_parameters() ->
controller_call(get_timetrap_parameters).
create_priv_dir(Value) ->
controller_call({create_priv_dir,Value}).
trc(TraceFile) ->
controller_call({trace,TraceFile}, 2*?ACCEPT_TIMEOUT).
stop_trace() ->
controller_call(stop_trace).
node_started(Node) ->
gen_server:cast(?MODULE, {node_started,Node}).
cover(App, Analyse) when is_atom(App) ->
cover(App, none, Analyse);
cover(CoverFile, Analyse) ->
cover(none, CoverFile, Analyse).
cover(App, CoverFile, Analyse) ->
{Excl,Incl,Cross} = read_cover_file(CoverFile),
CoverInfo = #cover{app=App,
file=CoverFile,
excl=Excl,
incl=Incl,
cross=Cross,
level=Analyse},
controller_call({cover,CoverInfo}).
cover(CoverInfo) ->
controller_call({cover,CoverInfo}).
cover_compile(App,File,Excl,Incl,Cross,Analyse,Stop) ->
cover_compile(#cover{app=App,
file=File,
excl=Excl,
incl=Incl,
cross=Cross,
level=Analyse,
stop=Stop}).
testcase_callback(ModFunc) ->
controller_call({testcase_callback,ModFunc}).
set_random_seed(Seed) ->
controller_call({set_random_seed,Seed}).
kill_slavenodes() ->
controller_call(kill_slavenodes).
get_hosts() ->
get(test_server_hosts).
%%--------------------------------------------------------------------
add_job(Name, TopCase) ->
add_job(Name, TopCase, []).
add_job(Name, TopCase, Skip) ->
SuiteName =
case Name of
"." -> "current_dir";
".." -> "parent_dir";
Other -> Other
end,
Dir = filename:absname(SuiteName),
controller_call({add_job,Dir,SuiteName,TopCase,Skip}).
controller_call(Arg) ->
case catch gen_server:call(?MODULE, Arg, infinity) of
{'EXIT',{{badarg,_},{gen_server,call,_}}} ->
exit(test_server_ctrl_not_running);
{'EXIT',Reason} ->
exit(Reason);
Other ->
Other
end.
controller_call(Arg, Timeout) ->
case catch gen_server:call(?MODULE, Arg, Timeout) of
{'EXIT',{{badarg,_},{gen_server,call,_}}} ->
exit(test_server_ctrl_not_running);
{'EXIT',Reason} ->
exit(Reason);
Other ->
Other
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% init([])
%%
%% init() is the init function of the test_server's gen_server.
%%
init([]) ->
case os:getenv("TEST_SERVER_CALL_TRACE") of
false ->
ok;
"" ->
ok;
TraceSpec ->
test_server_sup:call_trace(TraceSpec)
end,
process_flag(trap_exit, true),
%% copy format_exception setting from init arg to application environment
case init:get_argument(test_server_format_exception) of
{ok,[[TSFE]]} ->
application:set_env(test_server, format_exception, list_to_atom(TSFE));
_ ->
ok
end,
test_server_sup:cleanup_crash_dumps(),
test_server_sup:util_start(),
State = #state{jobs=[],finish=false},
TI0 = test_server:init_target_info(),
TargetHost = test_server_sup:hoststr(),
TI = TI0#target_info{host=TargetHost,
naming=naming(),
master=TargetHost},
_ = ets:new(slave_tab, [named_table,set,public,{keypos,2}]),
set_hosts([TI#target_info.host]),
{ok,State#state{target_info=TI}}.
naming() ->
case lists:member($., test_server_sup:hoststr()) of
true -> "-name";
false -> "-sname"
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(kill_slavenodes, From, State) -> ok
%%
%% Kill all slave nodes that remain after a test case
%% is completed.
%%
handle_call(kill_slavenodes, _From, State) ->
Nodes = test_server_node:kill_nodes(),
{reply, Nodes, State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({set_hosts, HostList}, From, State) -> ok
%%
%% Set the global hostlist.
%%
handle_call({set_hosts, Hosts}, _From, State) ->
set_hosts(Hosts),
{reply, ok, State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(get_hosts, From, State) -> [Hosts]
%%
%% Returns the lists of hosts that the test server
%% can use for slave nodes. This is primarily used
%% for nodename generation.
%%
handle_call(get_hosts, _From, State) ->
Hosts = get_hosts(),
{reply, Hosts, State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({add_job,Dir,Name,TopCase,Skip}, _, State) ->
%% ok | {error,Reason}
%%
%% Dir = string()
%% Name = string()
%% TopCase = term()
%% Skip = [SkipItem]
%% SkipItem = {Mod,Comment} | {Mod,Case,Comment} | {Mod,Cases,Comment}
%% Mod = Case = atom()
%% Comment = string()
%% Cases = [Case]
%%
%% Adds a job to the job queue. The name of the job is Name. A log directory
%% will be created in Dir/Name.logs. TopCase may be anything that
%% collect_cases/3 accepts, plus the following:
%%
%% {spec,SpecName} executes the named test suite specification file. Commands
%% in the file should be in the format accepted by do_spec_list/1.
%%
%% {command_line,SpecList} executes the list of specification instructions
%% supplied, which should be in the format accepted by do_spec_list/1.
handle_call({add_job,Dir,Name,TopCase,Skip}, _From, State) ->
LogDir = Dir ++ ?logdir_ext,
ExtraTools =
case State#state.cover of
false -> [];
CoverInfo -> [{cover,CoverInfo}]
end,
ExtraTools1 =
case State#state.random_seed of
undefined -> ExtraTools;
Seed -> [{random_seed,Seed}|ExtraTools]
end,
case lists:keysearch(Name, 1, State#state.jobs) of
false ->
case TopCase of
{spec,SpecName} ->
Pid = spawn_tester(
?MODULE, do_spec,
[SpecName,{State#state.multiply_timetraps,
State#state.scale_timetraps}],
LogDir, Name, State#state.levels,
State#state.reject_io_reqs,
State#state.create_priv_dir,
State#state.testcase_callback, ExtraTools1),
NewJobs = [{Name,Pid}|State#state.jobs],
{reply, ok, State#state{jobs=NewJobs}};
{command_line,SpecList} ->
Pid = spawn_tester(
?MODULE, do_spec_list,
[SpecList,{State#state.multiply_timetraps,
State#state.scale_timetraps}],
LogDir, Name, State#state.levels,
State#state.reject_io_reqs,
State#state.create_priv_dir,
State#state.testcase_callback, ExtraTools1),
NewJobs = [{Name,Pid}|State#state.jobs],
{reply, ok, State#state{jobs=NewJobs}};
TopCase ->
case State#state.get_totals of
{CliPid,Fun} ->
Result = count_test_cases(TopCase, Skip),
Fun(CliPid, Result),
{reply, ok, State};
_ ->
Cfg = make_config([]),
Pid = spawn_tester(
?MODULE, do_test_cases,
[TopCase,Skip,Cfg,
{State#state.multiply_timetraps,
State#state.scale_timetraps}],
LogDir, Name, State#state.levels,
State#state.reject_io_reqs,
State#state.create_priv_dir,
State#state.testcase_callback, ExtraTools1),
NewJobs = [{Name,Pid}|State#state.jobs],
{reply, ok, State#state{jobs=NewJobs}}
end
end;
_ ->
{reply,{error,name_already_in_use},State}
end;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(jobs, _, State) -> JobList
%% JobList = [{Name,Pid}, ...]
%% Name = string()
%% Pid = pid()
%%
%% Return the list of current jobs.
handle_call(jobs, _From, State) ->
{reply,State#state.jobs,State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({abort_current_testcase,Reason}, _, State) -> Result
%% Reason = term()
%% Result = ok | {error,no_testcase_running}
%%
%% Attempts to abort the test case that's currently running.
handle_call({abort_current_testcase,Reason}, _From, State) ->
case State#state.jobs of
[{_,Pid}|_] ->
Pid ! {abort_current_testcase,Reason,self()},
receive
{Pid,abort_current_testcase,Result} ->
{reply, Result, State}
after 10000 ->
{reply, {error,no_testcase_running}, State}
end;
_ ->
{reply, {error,no_testcase_running}, State}
end;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({finish,Fini}, _, State) -> {ok,Pid}
%% Fini = true | abort
%%
%% Tells the test_server to stop as soon as there are no test suites
%% running. Immediately if none are running. Abort is handled as soon
%% as current test finishes.
handle_call({finish,Fini}, _From, State) ->
case State#state.jobs of
[] ->
lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Fini) end,
State#state.idle_notify),
State2 = State#state{finish=false},
{stop,shutdown,{ok,self()}, State2};
_SomeJobs ->
State2 = State#state{finish=Fini},
{reply, {ok,self()}, State2}
end;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({idle_notify,Fun}, From, State) -> {ok,Pid}
%%
%% Lets a test client subscribe to receive a notification when the
%% test server becomes idle (can be used to syncronize jobs).
%% test_server calls Fun(From) when idle.
handle_call({idle_notify,Fun}, {Cli,_Ref}, State) ->
case State#state.jobs of
[] -> self() ! report_idle;
_ -> ok
end,
Subscribed = State#state.idle_notify,
{reply, {ok,self()}, State#state{idle_notify=[{Cli,Fun}|Subscribed]}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(start_get_totals, From, State) -> {ok,Pid}
%%
%% Switch on the mode where the test server will only
%% report back the number of tests it would execute
%% given some subsequent jobs.
handle_call({start_get_totals,Fun}, {Cli,_Ref}, State) ->
{reply, {ok,self()}, State#state{get_totals={Cli,Fun}}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(stop_get_totals, From, State) -> ok
%%
%% Lets a test client subscribe to receive a notification when the
%% test server becomes idle (can be used to syncronize jobs).
%% test_server calls Fun(From) when idle.
handle_call(stop_get_totals, {_Cli,_Ref}, State) ->
{reply, ok, State#state{get_totals=false}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(get_levels, _, State) -> {Show,Major,Minor}
%% Show = integer()
%% Major = integer()
%% Minor = integer()
%%
%% Returns a 3-tuple with the logging thresholds.
%% All output and information from a test suite is tagged with a detail
%% level. Lower values are more "important". Text that is output using
%% io:format or similar is automatically tagged with detail level 50.
%%
%% All output with detail level:
%% less or equal to Show is displayed on the screen (default 1)
%% less or equal to Major is logged in the major log file (default 19)
%% greater or equal to Minor is logged in the minor log files (default 10)
handle_call(get_levels, _From, State) ->
{reply,State#state.levels,State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({set_levels,Show,Major,Minor}, _, State) -> ok
%% Show = integer()
%% Major = integer()
%% Minor = integer()
%%
%% Sets the logging thresholds, see handle_call(get_levels,...) above.
handle_call({set_levels,Show,Major,Minor}, _From, State) ->
{reply,ok,State#state{levels={Show,Major,Minor}}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({reject_io_reqs,Bool}, _, State) -> ok
%% Bool = bool()
%%
%% May be used to switch off stdout printouts to the minor log file
handle_call({reject_io_reqs,Bool}, _From, State) ->
{reply,ok,State#state{reject_io_reqs=Bool}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({multiply_timetraps,N}, _, State) -> ok
%% N = integer() | infinity
%%
%% Multiplies all timetraps set by test cases with N
handle_call({multiply_timetraps,N}, _From, State) ->
{reply,ok,State#state{multiply_timetraps=N}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({scale_timetraps,Bool}, _, State) -> ok
%% Bool = true | false
%%
%% Specifies if test_server should scale the timetrap value
%% automatically if e.g. cover is running.
handle_call({scale_timetraps,Bool}, _From, State) ->
{reply,ok,State#state{scale_timetraps=Bool}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(get_timetrap_parameters, _, State) -> {Multiplier,Scale}
%% Multiplier = integer() | infinity
%% Scale = true | false
%%
%% Returns the parameter values that affect timetraps.
handle_call(get_timetrap_parameters, _From, State) ->
{reply,{State#state.multiply_timetraps,State#state.scale_timetraps},State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({trace,TraceFile}, _, State) -> ok | {error,Reason}
%%
%% Starts a separate node (trace control node) which
%% starts tracing on target and all slave nodes
%%
%% TraceFile is a text file with elements of type
%% {Trace,Mod,TracePattern}.
%% {Trace,Mod,Func,TracePattern}.
%% {Trace,Mod,Func,Arity,TracePattern}.
%%
%% Trace = tp | tpl; local or global call trace
%% Mod,Func = atom(), Arity=integer(); defines what to trace
%% TracePattern = [] | match_spec()
%%
%% The 'call' trace flag is set on all processes, and then
%% the given trace patterns are set.
handle_call({trace,TraceFile}, _From, State=#state{trc=false}) ->
TI = State#state.target_info,
case test_server_node:start_tracer_node(TraceFile, TI) of
{ok,Tracer} -> {reply,ok,State#state{trc=Tracer}};
Error -> {reply,Error,State}
end;
handle_call({trace,_TraceFile}, _From, State) ->
{reply,{error,already_tracing},State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(stop_trace, _, State) -> ok | {error,Reason}
%%
%% Stops tracing on target and all slave nodes and
%% terminates trace control node
handle_call(stop_trace, _From, State=#state{trc=false}) ->
{reply,{error,not_tracing},State};
handle_call(stop_trace, _From, State) ->
R = test_server_node:stop_tracer_node(State#state.trc),
{reply,R,State#state{trc=false}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({cover,CoverInfo}, _, State) -> ok | {error,Reason}
%%
%% Set specification of cover analysis to be used when running tests
%% (see start_extra_tools/1 and stop_extra_tools/1)
handle_call({cover,CoverInfo}, _From, State) ->
{reply,ok,State#state{cover=CoverInfo}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({create_priv_dir,Value}, _, State) -> ok | {error,Reason}
%%
%% Set create_priv_dir to either auto_per_run (create common priv dir once
%% per test run), manual_per_tc (the priv dir name will be unique for each
%% test case, but the user has to call test_server:make_priv_dir/0 to create
%% it), or auto_per_tc (unique priv dir created automatically for each test
%% case).
handle_call({create_priv_dir,Value}, _From, State) ->
{reply,ok,State#state{create_priv_dir=Value}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({testcase_callback,{Mod,Func}}, _, State) -> ok | {error,Reason}
%%
%% Add a callback function that will be called before and after every
%% test case (on the test case process):
%%
%% Mod:Func(Suite,TestCase,InitOrEnd,Config)
%%
%% InitOrEnd = init | 'end'.
handle_call({testcase_callback,ModFunc}, _From, State) ->
case ModFunc of
{Mod,Func} ->
_ = case code:is_loaded(Mod) of
{file,_} ->
ok;
false ->
code:load_file(Mod)
end,
case erlang:function_exported(Mod,Func,4) of
true ->
ok;
false ->
io:format(user,
"WARNING! Callback function ~w:~tw/4 undefined.~n~n",
[Mod,Func])
end;
_ ->
ok
end,
{reply,ok,State#state{testcase_callback=ModFunc}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({set_random_seed,Seed}, _, State) -> ok | {error,Reason}
%%
%% Let operator set a random seed value to be used e.g. for shuffling
%% test cases.
handle_call({set_random_seed,Seed}, _From, State) ->
{reply,ok,State#state{random_seed=Seed}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(stop, _, State) -> ok
%%
%% Stops the test server immediately.
%% Some cleanup is done by terminate/2
handle_call(stop, _From, State) ->
{stop, shutdown, ok, State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call(get_target_info, _, State) -> TI
%%
%% TI = #target_info{}
%%
%% Returns information about target
handle_call(get_target_info, _From, State) ->
{reply, State#state.target_info, State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({start_node,Name,Type,Options}, _, State) ->
%% ok | {error,Reason}
%%
%% Starts a new node (slave or peer)
handle_call({start_node, Name, Type, Options}, From, State) ->
%% test_server_ctrl does gen_server:reply/2 explicitly
test_server_node:start_node(Name, Type, Options, From,
State#state.target_info),
{noreply,State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({wait_for_node,Node}, _, State) -> ok
%%
%% Waits for a new node to take contact. Used if
%% node is started with option {wait,false}
handle_call({wait_for_node, Node}, From, State) ->
NewWaitList =
case ets:lookup(slave_tab,Node) of
[] ->
[{Node,From}|State#state.wait_for_node];
_ ->
gen_server:reply(From,ok),
State#state.wait_for_node
end,
{noreply,State#state{wait_for_node=NewWaitList}};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({stop_node,Name}, _, State) -> ok | {error,Reason}
%%
%% Stops a slave or peer node. This is actually only some cleanup
%% - the node is really stopped by test_server when this returns.
handle_call({stop_node, Name}, _From, State) ->
R = test_server_node:stop_node(Name),
{reply, R, State};
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_call({is_release_available,Name}, _, State) -> ok | {error,Reason}
%%
%% Tests if the release is available.
handle_call({is_release_available, Release}, _From, State) ->
R = test_server_node:is_release_available(Release),
{reply, R, State}.
%%--------------------------------------------------------------------
set_hosts(Hosts) ->
put(test_server_hosts, Hosts).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_cast({node_started,Name}, _, State)
%%
%% Called by test_server_node when a slave/peer node is fully started.
handle_cast({node_started,Node}, State) ->
case State#state.trc of
false -> ok;
Trc -> test_server_node:trace_nodes(Trc, [Node])
end,
NewWaitList =
case lists:keysearch(Node,1,State#state.wait_for_node) of
{value,{Node,From}} ->
gen_server:reply(From, ok),
lists:keydelete(Node, 1, State#state.wait_for_node);
false ->
State#state.wait_for_node
end,
{noreply, State#state{wait_for_node=NewWaitList}}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_info({'EXIT',Pid,Reason}, State)
%% Pid = pid()
%% Reason = term()
%%
%% Handles exit messages from linked processes. Only test suites are
%% expected to be linked. When a test suite terminates, it is removed
%% from the job queue.
handle_info(report_idle, State) ->
Finish = State#state.finish,
lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end,
State#state.idle_notify),
{noreply,State#state{idle_notify=[]}};
handle_info({'EXIT',Pid,Reason}, State) ->
case lists:keysearch(Pid,2,State#state.jobs) of
false ->
%% not our problem
{noreply,State};
{value,{Name,_}} ->
NewJobs = lists:keydelete(Pid, 2, State#state.jobs),
case Reason of
normal ->
fine;
killed ->
io:format("Suite ~ts was killed\n", [Name]);
_Other ->
io:format("Suite ~ts was killed with reason ~tp\n",
[Name,Reason])
end,
State2 = State#state{jobs=NewJobs},
Finish = State2#state.finish,
case NewJobs of
[] ->
lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end,
State2#state.idle_notify),
case Finish of
false ->
{noreply,State2#state{idle_notify=[]}};
_ -> % true | abort
%% test_server:finish() has been called and
%% there are no jobs in the job queue =>
%% stop the test_server_ctrl
{stop,shutdown,State2#state{finish=false}}
end;
_ -> % pending jobs
case Finish of
abort -> % abort test now!
lists:foreach(fun({Cli,Fun}) -> Fun(Cli,Finish) end,
State2#state.idle_notify),
{stop,shutdown,State2#state{finish=false}};
_ -> % true | false
{noreply, State2}
end
end
end;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% handle_info({tcp_closed,Sock}, State)
%%
%% A Socket was closed. This indicates that a node died.
%% This can be
%% *Slave or peer node started by a test suite
%% *Trace controll node
handle_info({tcp_closed,Sock}, State=#state{trc=Sock}) ->
%% Tracer node died - can't really do anything
%%! Maybe print something???
{noreply,State#state{trc=false}};
handle_info({tcp_closed,Sock}, State) ->
test_server_node:nodedown(Sock),
{noreply,State};
handle_info(_, State) ->
%% dummy; accept all, do nothing.
{noreply, State}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% terminate(Reason, State) -> ok
%% Reason = term()
%%
%% Cleans up when the test_server is terminating. Kills the running
%% test suites (if any) and any possible remainting slave node
terminate(_Reason, State) ->
test_server_sup:util_stop(),
case State#state.trc of
false -> ok;
Sock -> test_server_node:stop_tracer_node(Sock)
end,
ok = kill_all_jobs(State#state.jobs),
_ = test_server_node:kill_nodes(),
ok.
kill_all_jobs([{_Name,JobPid}|Jobs]) ->
exit(JobPid, kill),
kill_all_jobs(Jobs);
kill_all_jobs([]) ->
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%----------------------- INTERNAL FUNCTIONS -----------------------%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% spawn_tester(Mod, Func, Args, Dir, Name, Levels, RejectIoReqs,
%% CreatePrivDir, TestCaseCallback, ExtraTools) -> Pid
%% Mod = atom()
%% Func = atom()
%% Args = [term(),...]
%% Dir = string()
%% Name = string()
%% Levels = {integer(),integer(),integer()}
%% RejectIoReqs = bool()
%% CreatePrivDir = auto_per_run | manual_per_tc | auto_per_tc
%% TestCaseCallback = {CBMod,CBFunc} | undefined
%% ExtraTools = [ExtraTool,...]
%% ExtraTool = CoverInfo | TraceInfo | RandomSeed
%%
%% Spawns a test suite execute-process, just an ordinary spawn, except
%% that it will set a lot of dictionary information before starting the
%% named function. Also, the execution is timed and protected by a catch.
%% When the named function is done executing, a summary of the results
%% is printed to the log files.
spawn_tester(Mod, Func, Args, Dir, Name, Levels, RejectIoReqs,
CreatePrivDir, TCCallback, ExtraTools) ->
spawn_link(fun() ->
init_tester(Mod, Func, Args, Dir, Name, Levels, RejectIoReqs,
CreatePrivDir, TCCallback, ExtraTools)
end).
init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels,
RejectIoReqs, CreatePrivDir, TCCallback, ExtraTools) ->
process_flag(trap_exit, true),
_ = test_server_io:start_link(),
put(app, common_test),
put(test_server_name, Name),
put(test_server_dir, Dir),
put(test_server_total_time, 0),
put(test_server_ok, 0),
put(test_server_failed, 0),
put(test_server_skipped, {0,0}),
put(test_server_minor_level, MinLev),
put(test_server_create_priv_dir, CreatePrivDir),
put(test_server_random_seed, proplists:get_value(random_seed, ExtraTools)),
put(test_server_testcase_callback, TCCallback),
case os:getenv("TEST_SERVER_FRAMEWORK") of
FW when FW =:= false; FW =:= "undefined" ->
put(test_server_framework, '$none');
FW ->
put(test_server_framework_name, list_to_atom(FW)),
case os:getenv("TEST_SERVER_FRAMEWORK_NAME") of
FWName when FWName =:= false; FWName =:= "undefined" ->
put(test_server_framework_name, '$none');
FWName ->
put(test_server_framework_name, list_to_atom(FWName))
end
end,
%% before first print, read and set logging options
FWLogDir =
case test_server_sup:framework_call(get_log_dir, [], []) of
{ok,FwDir} -> FwDir;
_ -> filename:dirname(Dir)
end,
put(test_server_framework_logdir, FWLogDir),
LogOpts = test_server_sup:framework_call(get_logopts, [], []),
put(test_server_logopts, LogOpts),
StartedExtraTools = start_extra_tools(ExtraTools),
test_server_io:set_job_name(Name),
test_server_io:set_gl_props([{levels,Levels},
{auto_nl,not lists:member(no_nl, LogOpts)},
{reject_io_reqs,RejectIoReqs}]),
group_leader(test_server_io:get_gl(true), self()),
{TimeMy,Result} = ts_tc(Mod, Func, Args),
set_io_buffering(undefined),
test_server_io:set_job_name(undefined),
catch stop_extra_tools(StartedExtraTools),
case Result of
{'EXIT',test_suites_done} ->
ok;
{'EXIT',_Pid,Reason} ->
print(1, "EXIT, reason ~tp", [Reason]);
{'EXIT',Reason} ->
report_severe_error(Reason),
print(1, "EXIT, reason ~tp", [Reason])
end,
Time = TimeMy/1000000,
SuccessStr =
case get(test_server_failed) of
0 -> "Ok";
_ -> "FAILED"
end,
{SkippedN,SkipStr} =
case get(test_server_skipped) of
{0,0} ->
{0,""};
{USkipped,ASkipped} ->
Skipped = USkipped+ASkipped,
{Skipped,io_lib:format(", ~w Skipped", [Skipped])}
end,
OkN = get(test_server_ok),
FailedN = get(test_server_failed),
print(html,"\n\n\n"
"
TOTAL
"
"
~.3fs
~ts
~w Ok, ~w Failed~ts of ~w
\n"
"\n",
[Time,SuccessStr,OkN,FailedN,SkipStr,OkN+FailedN+SkippedN]),
test_server_io:stop([major,html,unexpected_io]),
{UnexpectedIoName,UnexpectedIoFooter} = get(test_server_unexpected_footer),
{ok,UnexpectedIoFd} = open_html_file(UnexpectedIoName, [append]),
io:put_chars(UnexpectedIoFd, "\n\n"++UnexpectedIoFooter),
ok = file:close(UnexpectedIoFd).
report_severe_error(Reason) ->
test_server_sup:framework_call(report, [severe_error,Reason]).
ts_tc(M,F,A) ->
Before = erlang:monotonic_time(),
Result = (catch apply(M, F, A)),
After = erlang:monotonic_time(),
Elapsed = erlang:convert_time_unit(After-Before,
native,
micro_seconds),
{Elapsed, Result}.
start_extra_tools(ExtraTools) ->
start_extra_tools(ExtraTools, []).
start_extra_tools([{cover,CoverInfo} | ExtraTools], Started) ->
case start_cover(CoverInfo) of
{ok,NewCoverInfo} ->
start_extra_tools(ExtraTools,[{cover,NewCoverInfo}|Started]);
{error,_} ->
start_extra_tools(ExtraTools, Started)
end;
start_extra_tools([_ | ExtraTools], Started) ->
start_extra_tools(ExtraTools, Started);
start_extra_tools([], Started) ->
Started.
stop_extra_tools(ExtraTools) ->
TestDir = get(test_server_log_dir_base),
case lists:keymember(cover, 1, ExtraTools) of
false ->
write_default_coverlog(TestDir);
true ->
ok
end,
stop_extra_tools(ExtraTools, TestDir).
stop_extra_tools([{cover,CoverInfo}|ExtraTools], TestDir) ->
stop_cover(CoverInfo,TestDir),
stop_extra_tools(ExtraTools, TestDir);
%%stop_extra_tools([_ | ExtraTools], TestDir) ->
%% stop_extra_tools(ExtraTools, TestDir);
stop_extra_tools([], _) ->
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% do_spec(SpecName, TimetrapSpec) -> {error,Reason} | exit(Result)
%% SpecName = string()
%% TimetrapSpec = MultiplyTimetrap | {MultiplyTimetrap,ScaleTimetrap}
%% MultiplyTimetrap = integer() | infinity
%% ScaleTimetrap = bool()
%%
%% Reads the named test suite specification file, and executes it.
%%
%% This function is meant to be called by a process created by
%% spawn_tester/10, which sets up some necessary dictionary values.
do_spec(SpecName, TimetrapSpec) when is_list(SpecName) ->
case file:consult(SpecName) of
{ok,TermList} ->
do_spec_list(TermList,TimetrapSpec);
{error,Reason} ->
io:format("Can't open ~ts: ~tp\n", [SpecName,Reason]),
{error,{cant_open_spec,Reason}}
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% do_spec_list(TermList, TimetrapSpec) -> exit(Result)
%% TermList = [term()|...]
%% TimetrapSpec = MultiplyTimetrap | {MultiplyTimetrap,ScaleTimetrap}
%% MultiplyTimetrap = integer() | infinity
%% ScaleTimetrap = bool()
%%
%% Executes a list of test suite specification commands. The following
%% commands are available, and may occur zero or more times (if several,
%% the contents is appended):
%%
%% {topcase,TopCase} Specifies top level test goals. TopCase has the syntax
%% specified by collect_cases/3.
%%
%% {skip,Skip} Specifies test cases to skip, and lists requirements that
%% cannot be granted during the test run. Skip has the syntax specified
%% by collect_cases/3.
%%
%% {nodes,Nodes} Lists node names avaliable to the test suites. Nodes have
%% the syntax specified by collect_cases/3.
%%
%% {require_nodenames, Num} Specifies how many nodenames the test suite will
%% need. Theese are automaticly generated and inserted into the Config by the
%% test_server. The caller may specify other hosts to run theese nodes by
%% using the {hosts, Hosts} option. If there are no hosts specified, all
%% nodenames will be generated from the local host.
%%
%% {hosts, Hosts} Specifies a list of available hosts on which to start
%% slave nodes. It is used when the {remote, true} option is given to the
%% test_server:start_node/3 function. Also, if {require_nodenames, Num} is
%% contained in the TermList, the generated nodenames will be spread over
%% all hosts given in this Hosts list. The hostnames are given as atoms or
%% strings.
%%
%% {diskless, true} is kept for backwards compatiblilty and
%% should not be used. Use a configuration test case instead.
%%
%% This function is meant to be called by a process created by
%% spawn_tester/10, which sets up some necessary dictionary values.
do_spec_list(TermList0, TimetrapSpec) ->
Nodes = [],
TermList =
case lists:keysearch(hosts, 1, TermList0) of
{value, {hosts, Hosts0}} ->
Hosts = lists:map(fun(H) -> cast_to_list(H) end, Hosts0),
controller_call({set_hosts, Hosts}),
lists:keydelete(hosts, 1, TermList0);
_ ->
TermList0
end,
DefaultConfig = make_config([{nodes,Nodes}]),
{TopCases,SkipList,Config} = do_spec_terms(TermList, [], [], DefaultConfig),
do_test_cases(TopCases, SkipList, Config, TimetrapSpec).
do_spec_terms([], TopCases, SkipList, Config) ->
{TopCases,SkipList,Config};
do_spec_terms([{topcase,TopCase}|Terms], TopCases, SkipList, Config) ->
do_spec_terms(Terms,[TopCase|TopCases], SkipList, Config);
do_spec_terms([{skip,Skip}|Terms], TopCases, SkipList, Config) ->
do_spec_terms(Terms, TopCases, [Skip|SkipList], Config);
do_spec_terms([{nodes,Nodes}|Terms], TopCases, SkipList, Config) ->
do_spec_terms(Terms, TopCases, SkipList,
update_config(Config, {nodes,Nodes}));
do_spec_terms([{diskless,How}|Terms], TopCases, SkipList, Config) ->
do_spec_terms(Terms, TopCases, SkipList,
update_config(Config, {diskless,How}));
do_spec_terms([{config,MoreConfig}|Terms], TopCases, SkipList, Config) ->
do_spec_terms(Terms, TopCases, SkipList, Config++MoreConfig);
do_spec_terms([{default_timeout,Tmo}|Terms], TopCases, SkipList, Config) ->
do_spec_terms(Terms, TopCases, SkipList,
update_config(Config, {default_timeout,Tmo}));
do_spec_terms([{require_nodenames,NumNames}|Terms], TopCases, SkipList, Config) ->
NodeNames0=generate_nodenames(NumNames),
NodeNames=lists:delete([], NodeNames0),
do_spec_terms(Terms, TopCases, SkipList,
update_config(Config, {nodenames,NodeNames}));
do_spec_terms([Other|Terms], TopCases, SkipList, Config) ->
io:format("** WARNING: Spec file contains unknown directive ~tp\n",
[Other]),
do_spec_terms(Terms, TopCases, SkipList, Config).
generate_nodenames(Num) ->
Hosts = case controller_call(get_hosts) of
[] ->
TI = controller_call(get_target_info),
[TI#target_info.host];
List ->
List
end,
generate_nodenames2(Num, Hosts, []).
generate_nodenames2(0, _Hosts, Acc) ->
Acc;
generate_nodenames2(N, Hosts, Acc) ->
Host=lists:nth((N rem (length(Hosts)))+1, Hosts),
Name=list_to_atom(temp_nodename("nod", []) ++ "@" ++ Host),
generate_nodenames2(N-1, Hosts, [Name|Acc]).
temp_nodename([], Acc) ->
lists:flatten(Acc);
temp_nodename([Chr|Base], Acc) ->
{A,B,C} = ?now,
New = [Chr | integer_to_list(Chr bxor A bxor B+A bxor C+B)],
temp_nodename(Base, [New|Acc]).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% count_test_cases(TopCases, SkipCases) -> {Suites,NoOfCases} | error
%% TopCases = term() (See collect_cases/3)
%% SkipCases = term() (See collect_cases/3)
%% Suites = list()
%% NoOfCases = integer() | unknown
%%
%% Counts the test cases that are about to run and returns that number.
%% If there's a conf group in TestSpec with a repeat property, the total number
%% of cases can not be calculated and NoOfCases = unknown.
count_test_cases(TopCases, SkipCases) when is_list(TopCases) ->
case collect_all_cases(TopCases, SkipCases) of
{error,_Why} = Error ->
Error;
TestSpec ->
{get_suites(TestSpec, []),
case remove_conf(TestSpec) of
{repeats,_} ->
unknown;
TestSpec1 ->
length(TestSpec1)
end}
end;
count_test_cases(TopCase, SkipCases) ->
count_test_cases([TopCase], SkipCases).
remove_conf(Cases) ->
remove_conf(Cases, [], false).
remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) ->
case get_repeat(Props) of
undefined ->
remove_conf(Cases, NoConf, Repeats);
{_RepType,1} ->
remove_conf(Cases, NoConf, Repeats);
_ ->
remove_conf(Cases, NoConf, true)
end;
remove_conf([{make,_Ref,_MF}|Cases], NoConf, Repeats) ->
remove_conf(Cases, NoConf, Repeats);
remove_conf([{skip_case,{{_M,all},_Cmt},_Mode}|Cases], NoConf, Repeats) ->
remove_conf(Cases, NoConf, Repeats);
remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt}}|Cases],
NoConf, Repeats) when Type==conf;
Type==make ->
remove_conf(Cases, NoConf, Repeats);
remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt},_Mode}|Cases],
NoConf, Repeats) when Type==conf;
Type==make ->
remove_conf(Cases, NoConf, Repeats);
remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) ->
FwMod = get_fw_mod(?MODULE),
if Mod == FwMod ->
remove_conf(Cases, NoConf, Repeats);
true ->
remove_conf(Cases, [C|NoConf], Repeats)
end;
remove_conf([C={repeat,_,_}|Cases], NoConf, _Repeats) ->
remove_conf(Cases, [C|NoConf], true);
remove_conf([C|Cases], NoConf, Repeats) ->
remove_conf(Cases, [C|NoConf], Repeats);
remove_conf([], NoConf, true) ->
{repeats,lists:reverse(NoConf)};
remove_conf([], NoConf, false) ->
lists:reverse(NoConf).
get_suites([{skip_case,{{Mod,_F},_Cmt},_Mode}|Tests], Mods) when is_atom(Mod) ->
case add_mod(Mod, Mods) of
true -> get_suites(Tests, [Mod|Mods]);
false -> get_suites(Tests, Mods)
end;
get_suites([{Mod,_Case}|Tests], Mods) when is_atom(Mod) ->
case add_mod(Mod, Mods) of
true -> get_suites(Tests, [Mod|Mods]);
false -> get_suites(Tests, Mods)
end;
get_suites([{Mod,_Func,_Args}|Tests], Mods) when is_atom(Mod) ->
case add_mod(Mod, Mods) of
true -> get_suites(Tests, [Mod|Mods]);
false -> get_suites(Tests, Mods)
end;
get_suites([_|Tests], Mods) ->
get_suites(Tests, Mods);
get_suites([], Mods) ->
lists:reverse(Mods).
add_mod(Mod, Mods) ->
case lists:reverse(atom_to_list(Mod)) of
"ETIUS_" ++ _ -> % test suite
case lists:member(Mod, Mods) of
true -> false;
false -> true
end;
_ ->
false
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% do_test_cases(TopCases, SkipCases, Config, TimetrapSpec) ->
%% exit(Result)
%%
%% TopCases = term() (See collect_cases/3)
%% SkipCases = term() (See collect_cases/3)
%% Config = term() (See collect_cases/3)
%% TimetrapSpec = MultiplyTimetrap | {MultiplyTimetrap,ScaleTimetrap}
%% MultiplyTimetrap = integer() | infinity
%% ScaleTimetrap = bool()
%%
%% Initializes and starts the test run, for "ordinary" test suites.
%% Creates log directories and log files, inserts initial timestamps and
%% configuration information into the log files.
%%
%% This function is meant to be called by a process created by
%% spawn_tester/10, which sets up some necessary dictionary values.
do_test_cases(TopCases, SkipCases,
Config, MultiplyTimetrap) when is_integer(MultiplyTimetrap);
MultiplyTimetrap == infinity ->
do_test_cases(TopCases, SkipCases, Config, {MultiplyTimetrap,true});
do_test_cases(TopCases, SkipCases,
Config, TimetrapData) when is_list(TopCases),
is_tuple(TimetrapData) ->
{ok,TestDir} = start_log_file(),
FwMod = get_fw_mod(?MODULE),
case collect_all_cases(TopCases, SkipCases) of
{error,Why} ->
print(1, "Error starting: ~tp", [Why]),
exit(test_suites_done);
TestSpec0 ->
N = case remove_conf(TestSpec0) of
{repeats,_} -> unknown;
TS -> length(TS)
end,
put(test_server_cases, N),
put(test_server_case_num, 0),
TestSpec =
add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod),
TI = get_target_info(),
print(1, "Starting test~ts",
[print_if_known(N, {", ~w test cases",[N]},
{" (with repeated test cases)",[]})]),
Test = get(test_server_name),
TestName = if is_list(Test) ->
lists:flatten(io_lib:format("~ts", [Test]));
true ->
lists:flatten(io_lib:format("~tp", [Test]))
end,
TestDescr = "Test " ++ TestName ++ " results",
test_server_sup:framework_call(report, [tests_start,{Test,N}]),
{Header,Footer} =
case test_server_sup:framework_call(get_html_wrapper,
[TestDescr,true,TestDir,
{[],[2,3,4,7,8],[1,6]}], "") of
Empty when (Empty == "") ; (element(2,Empty) == "") ->
put(basic_html, true),
{[html_header(TestDescr),
"