1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2003-2020. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20
21%%----------------------------------------------------------------------
22%% Purpose: Lightweight test server
23%%----------------------------------------------------------------------
24
25-module(snmp_test_server).
26
27-compile(export_all).
28
29-export([
30	 run/1, run/2,
31
32	 error/3,
33	 skip/3,
34	 fatal_skip/3,
35
36	 init_per_testcase/2,
37	 end_per_testcase/2
38	]).
39
40-include("snmp_test_lib.hrl").
41
42-define(GLOBAL_LOGGER, snmp_global_logger).
43-define(TEST_CASE_SUP, snmp_test_case_supervisor).
44
45-define(d(F,A),d(F,A,?LINE)).
46
47-ifndef(snmp_priv_dir).
48-define(snmp_priv_dir, "run-" ++ timestamp()).
49-endif.
50
51
52%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
53%% Evaluates a test case or test suite
54%% Returns a list of failing test cases:
55%%
56%%     {Mod, Fun, ExpectedRes, ActualRes}
57%%----------------------------------------------------------------------
58
59run([Mod, Fun]) when is_atom(Mod) andalso is_atom(Fun) ->
60    Res = run({Mod, Fun}, default_config(Mod)),
61    display_result(Res),
62    Res;
63run({Mod, _Fun} = Case) when is_atom(Mod) ->
64    io:format("~n", []),
65    Res = run(Case, default_config(Mod)),
66    display_result(Res),
67    Res;
68run(Mod) when is_atom(Mod) ->
69    io:format("~n", []),
70    Res = run(Mod, default_config(Mod)),
71    display_result(Res),
72    Res;
73run([Mod]) when is_atom(Mod) ->
74    io:format("~n", []),
75    Res = run(Mod, default_config(Mod)),
76    display_result(Res),
77    Res.
78
79
80run({Mod, Fun}, Config) when is_atom(Mod) andalso
81			     is_atom(Fun) andalso
82			     is_list(Config) ->
83    ?d("run(~p,~p) -> entry", [Mod, Fun]),
84    case (catch apply(Mod, Fun, [suite])) of
85	[] ->
86	    io:format("~n~n*** Eval: ~p ***************~n",
87		      [{Mod, Fun}]),
88	    case eval(Mod, Fun, Config) of
89		{ok, _, _} ->
90		    [];
91		Other ->
92		    [Other]
93	    end;
94
95	Cases when is_list(Cases) ->
96	    io:format("~n*** Expand: ~p ...~n", [{Mod, Fun}]),
97	    Map = fun(Case) when is_atom(Case) -> {Mod, Case};
98		     (Case) -> Case
99		  end,
100	    run(lists:map(Map, Cases), Config);
101
102        {conf, InitSuite, Cases, FinishSuite} when is_atom(InitSuite) andalso
103						   is_list(Cases) andalso
104						   is_atom(FinishSuite) ->
105	    ?d("run -> conf: "
106	       "~n   InitSuite:   ~p"
107	       "~n   Cases:       ~p"
108	       "~n   FinishSuite: ~p", [InitSuite, Cases, FinishSuite]),
109	    do_suite(Mod, InitSuite, Cases, FinishSuite, Config);
110
111        {req, _, SubCases} when is_list(SubCases) ->
112	    ?d("run -> req: "
113	       "~n   SubCases: ~p", [SubCases]),
114	    do_subcases(Mod, Fun, SubCases, Config, []);
115
116        {req, _, Conf} ->
117	    ?d("run -> req: "
118	       "~n   Conf: ~p", [Conf]),
119	    do_subcases(Mod, Fun, [Conf], Config, []);
120
121        {'EXIT', {undef, _}} ->
122            io:format("~n*** Undefined:   ~p~n", [{Mod, Fun}]),
123            [{nyi, {Mod, Fun}, ok}];
124
125        Error ->
126            io:format("~n*** Ignoring:   ~p: ~p~n", [{Mod, Fun}, Error]),
127            [{failed, {Mod, Fun}, Error}]
128    end;
129
130run(Mod, Config) when is_atom(Mod) andalso is_list(Config) ->
131    run({Mod, all}, Config);
132
133run(Cases, Config) when is_list(Cases) andalso is_list(Config) ->
134    Errors = [run(Case, Config) || Case <- Cases],
135    lists:append(Errors);
136
137run(Bad, _Config) ->
138    [{badarg, Bad, ok}].
139
140
141do_suite(Mod, Init, Cases, Finish, Config0) ->
142    ?d("do_suite -> entry with"
143       "~n   Mod:     ~p"
144       "~n   Init:    ~p"
145       "~n   Cases:   ~p"
146       "~n   Finish:  ~p"
147       "~n   Config0: ~p", [Mod, Init, Cases, Finish, Config0]),
148    case (catch apply(Mod, Init, [Config0])) of
149	Config when is_list(Config) ->
150	    io:format("~n*** Expand: ~p ...~n", [Mod]),
151	    Map = fun(Case) when is_atom(Case) -> {Mod, Case};
152		     (Case) -> Case
153		  end,
154	    Res = run(lists:map(Map, Cases), Config),
155	    (catch apply(Mod, Finish, [Config])),
156	    Res;
157
158	{'EXIT', {skipped, Reason}} ->
159	    io:format(" => skipping: ~p~n", [Reason]),
160	    SkippedCases =
161		[{skipped, {Mod, Case}, suite_init_skipped} || Case <- Cases],
162	    lists:flatten([[{skipped, {Mod, Init}, Reason}],
163			   SkippedCases,
164			   [{skipped, {Mod, Finish}, suite_init_skipped}]]);
165
166	Error ->
167	    io:format(" => init (~p) failed: ~n~p~n", [Init, Error]),
168	    InitResult =
169		[{failed, {Mod, Init}, Error}],
170	    SkippedCases =
171		[{skipped, {Mod, Case}, suite_init_failed} || Case <- Cases],
172	    FinResult =
173		case (catch apply(Mod, Finish, [Config0])) of
174		    ok ->
175			[];
176		    FinConfig when is_list(FinConfig) ->
177			[];
178		    FinError ->
179			[{failed, {Mod, Finish}, FinError}]
180		end,
181	    lists:flatten([InitResult, SkippedCases, FinResult])
182
183    end.
184
185
186do_subcases(_Mod, _Fun, [], _Config, Acc) ->
187    lists:flatten(lists:reverse(Acc));
188do_subcases(Mod, Fun, [{conf, Init, Cases, Finish}|SubCases], Config, Acc)
189  when is_atom(Init) andalso is_list(Cases) andalso is_atom(Finish) ->
190    R = case (catch apply(Mod, Init, [Config])) of
191	    Conf when is_list(Conf) ->
192		io:format("~n*** Expand: ~p ...~n", [{Mod, Fun}]),
193		Map = fun(Case) when is_atom(Case) -> {Mod, Case};
194			 (Case) -> Case
195		      end,
196		Res = run(lists:map(Map, Cases), Conf),
197		(catch apply(Mod, Finish, [Conf])),
198		Res;
199
200	    {'EXIT', {skipped, Reason}} ->
201		io:format(" => skipping: ~p~n", [Reason]),
202		[{skipped, {Mod, Fun}, Reason}];
203
204	    Error ->
205		io:format(" => init (~p) failed: ~n~p~n", [Init, Error]),
206		(catch apply(Mod, Finish, [Config])),
207		[{failed, {Mod, Fun}, Error}]
208	end,
209    do_subcases(Mod, Fun, SubCases, Config, [R|Acc]);
210do_subcases(Mod, Fun, [SubCase|SubCases], Config, Acc) when is_atom(SubCase) ->
211    R = do_case(Mod, SubCase, Config),
212    do_subcases(Mod, Fun, SubCases,Config, [R|Acc]).
213
214
215do_case(M, F, C) ->
216    io:format("~n~n*** Eval: ~p ***************~n", [{M, F}]),
217    case eval(M, F, C) of
218	{ok, _, _} ->
219	    [];
220	Other ->
221	    [Other]
222    end.
223
224
225eval(Mod, Fun, Config) ->
226    Flag    = process_flag(trap_exit, true),
227    global:register_name(?TEST_CASE_SUP, self()),
228    Config2 = Mod:init_per_testcase(Fun, Config),
229    Self    = self(),
230    Eval    = fun() -> do_eval(Self, Mod, Fun, Config2) end,
231    Pid     = spawn_link(Eval),
232    R       = wait_for_evaluator(Pid, Mod, Fun, Config2, []),
233    Mod:end_per_testcase(Fun, Config2),
234    global:unregister_name(?TEST_CASE_SUP),
235    process_flag(trap_exit, Flag),
236    R.
237
238wait_for_evaluator(Pid, Mod, Fun, Config, Errors) ->
239    Pre = lists:concat(["TEST CASE: ", Fun]),
240    receive
241	{'EXIT', _Watchdog, watchdog_timeout} ->
242	    io:format("*** ~s WATCHDOG TIMEOUT~n", [Pre]),
243	    exit(Pid, kill),
244	    {failed, {Mod, Fun}, watchdog_timeout};
245	{done, Pid, ok} when Errors =:= [] ->
246	    io:format("*** ~s OK~n", [Pre]),
247	    {ok, {Mod, Fun}, Errors};
248	{done, Pid, {ok, _}} when Errors =:= [] ->
249	    io:format("*** ~s OK~n", [Pre]),
250	    {ok, {Mod, Fun}, Errors};
251	{done, Pid, Fail} ->
252	    io:format("*** ~s FAILED~n~p~n", [Pre, Fail]),
253	    {failed, {Mod, Fun}, Fail};
254	{'EXIT', Pid, {skipped, Reason}} ->
255	    io:format("*** ~s SKIPPED~n~p~n", [Pre, Reason]),
256	    {skipped, {Mod, Fun}, Errors};
257	{'EXIT', Pid, Reason} ->
258	    io:format("*** ~s CRASHED~n~p~n", [Pre, Reason]),
259	    {crashed, {Mod, Fun}, [{'EXIT', Reason} | Errors]};
260	{fail, Pid, Reason} ->
261	    io:format("*** ~s FAILED~n~p~n", [Pre, Reason]),
262	    wait_for_evaluator(Pid, Mod, Fun, Config, Errors ++ [Reason])
263   end.
264
265do_eval(ReplyTo, Mod, Fun, Config) ->
266    case (catch apply(Mod, Fun, [Config])) of
267	{'EXIT', {skipped, Reason}} ->
268	    ReplyTo ! {'EXIT', self(), {skipped, Reason}};
269	Other ->
270	    ReplyTo ! {done, self(), Other}
271    end,
272    unlink(ReplyTo),
273    exit(shutdown).
274
275
276display_result([]) ->
277    io:format("TEST OK~n", []);
278
279display_result(Errors) when is_list(Errors) ->
280    Nyi     = [MF || {nyi, MF, _} <- Errors],
281    Skipped = [{MF, Reason} || {skipped, MF, Reason} <- Errors],
282    Crashed = [{MF, Reason} || {crashed, MF, Reason} <- Errors],
283    Failed  = [{MF, Reason} || {failed,  MF, Reason} <- Errors],
284    display_skipped(Skipped),
285    display_crashed(Crashed),
286    display_failed(Failed),
287    display_summery(Nyi, Skipped, Crashed, Failed).
288
289display_summery(Nyi, Skipped, Crashed, Failed) ->
290    io:format("~nTest case summery:~n", []),
291    display_summery(Nyi, "not yet implemented"),
292    display_summery(Skipped, "skipped"),
293    display_summery(Crashed, "crashed"),
294    display_summery(Failed, "failed"),
295    io:format("~n", []).
296
297display_summery([], _) ->
298    ok;
299display_summery(Res, Info) ->
300    io:format("  ~w test cases ~s~n", [length(Res), Info]).
301
302display_skipped([]) ->
303    io:format("Skipped test cases: -~n", []);
304display_skipped(Skipped) ->
305    io:format("Skipped test cases:~n", []),
306    [io:format("  ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Skipped],
307    io:format("~n", []).
308
309display_crashed([]) ->
310    io:format("Crashed test cases: -~n", []);
311display_crashed(Crashed) ->
312    io:format("Crashed test cases:~n", []),
313    [io:format("  ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Crashed],
314    io:format("~n", []).
315
316display_failed([]) ->
317    io:format("Failed test cases: -~n", []);
318display_failed(Failed) ->
319    io:format("Failed test cases:~n", []),
320    [io:format("  ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Failed],
321    io:format("~n", []).
322
323
324%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
325%% Verify that the actual result of a test case matches the exected one
326%% Returns the actual result
327%% Stores the result in the process dictionary if mismatch
328
329error(Actual, Mod, Line) ->
330    global:send(?GLOBAL_LOGGER, {failed, Mod, Line}),
331    log("<ERROR> Bad result: ~p~n", [Actual], Mod, Line),
332    case global:whereis_name(?TEST_CASE_SUP) of
333	undefined ->
334	    ignore;
335	Pid ->
336	    Pid ! {fail, self(), {Actual, Mod, Line}}
337    end,
338    Actual.
339
340skip(Actual, Mod, Line) ->
341    log("Skipping test case~n", [], Mod, Line),
342    exit({skipped, {Actual, Mod, Line}}).
343
344fatal_skip(Actual, Mod, Line) ->
345    error(Actual, Mod, Line),
346    exit(shutdown).
347
348
349log(Format, Args, Mod, Line) ->
350    case global:whereis_name(?GLOBAL_LOGGER) of
351	undefined ->
352	    io:format(user, "~p(~p): " ++ Format, [Mod, Line] ++ Args);
353	Pid ->
354	    io:format(Pid, "~p(~p): " ++ Format, [Mod, Line] ++ Args)
355    end.
356
357
358%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
359%% Test server callbacks
360
361init_per_testcase(_Case, Config) ->
362    global:register_name(?GLOBAL_LOGGER, group_leader()),
363    Config.
364
365end_per_testcase(_Case, _Config) ->
366    global:unregister_name(?GLOBAL_LOGGER),
367    ok.
368
369
370%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
371%% Internal utility functions
372
373default_config(Mod) ->
374    PrivDir0 = ?snmp_priv_dir,
375    case filename:pathtype(PrivDir0) of
376	absolute ->
377	    ok;
378	_ ->
379	    case file:make_dir(Mod) of
380		ok ->
381		    ok;
382		{error, eexist} ->
383		    ok
384	    end,
385	    PrivDir = filename:join(Mod, PrivDir0),
386	    case file:make_dir(PrivDir) of
387		ok ->
388		    ok;
389		{error, eexist} ->
390		    ok;
391		Error ->
392		    ?FAIL({failed_creating_subsuite_top_dir, Error})
393	    end,
394	    [{priv_dir, PrivDir}]
395    end.
396
397
398%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
399
400d(F, A, L) ->
401    d(true, F, A, L).
402    %% d(get(dbg), F, A, L).
403
404d(true, F, A, L) ->
405    io:format("STS[~w] ~p " ++ F ++ "~n", [L,self()|A]);
406d(_, _, _, _) ->
407    ok.
408
409timestamp() ->
410    {Date, Time}     = calendar:now_to_datetime( erlang:timestamp() ),
411    {YYYY, MM, DD}   = Date,
412    {Hour, Min, Sec} = Time,
413    FormatDate =
414        io_lib:format("~.4w-~.2.0w-~.2.0w_~.2.0w.~.2.0w.~.2.0w",
415                      [YYYY,MM,DD,Hour,Min,Sec]),
416    lists:flatten(FormatDate).
417
418