1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2003-2020. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20 21%%---------------------------------------------------------------------- 22%% Purpose: Lightweight test server 23%%---------------------------------------------------------------------- 24 25-module(snmp_test_server). 26 27-compile(export_all). 28 29-export([ 30 run/1, run/2, 31 32 error/3, 33 skip/3, 34 fatal_skip/3, 35 36 init_per_testcase/2, 37 end_per_testcase/2 38 ]). 39 40-include("snmp_test_lib.hrl"). 41 42-define(GLOBAL_LOGGER, snmp_global_logger). 43-define(TEST_CASE_SUP, snmp_test_case_supervisor). 44 45-define(d(F,A),d(F,A,?LINE)). 46 47-ifndef(snmp_priv_dir). 48-define(snmp_priv_dir, "run-" ++ timestamp()). 49-endif. 50 51 52%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 53%% Evaluates a test case or test suite 54%% Returns a list of failing test cases: 55%% 56%% {Mod, Fun, ExpectedRes, ActualRes} 57%%---------------------------------------------------------------------- 58 59run([Mod, Fun]) when is_atom(Mod) andalso is_atom(Fun) -> 60 Res = run({Mod, Fun}, default_config(Mod)), 61 display_result(Res), 62 Res; 63run({Mod, _Fun} = Case) when is_atom(Mod) -> 64 io:format("~n", []), 65 Res = run(Case, default_config(Mod)), 66 display_result(Res), 67 Res; 68run(Mod) when is_atom(Mod) -> 69 io:format("~n", []), 70 Res = run(Mod, default_config(Mod)), 71 display_result(Res), 72 Res; 73run([Mod]) when is_atom(Mod) -> 74 io:format("~n", []), 75 Res = run(Mod, default_config(Mod)), 76 display_result(Res), 77 Res. 78 79 80run({Mod, Fun}, Config) when is_atom(Mod) andalso 81 is_atom(Fun) andalso 82 is_list(Config) -> 83 ?d("run(~p,~p) -> entry", [Mod, Fun]), 84 case (catch apply(Mod, Fun, [suite])) of 85 [] -> 86 io:format("~n~n*** Eval: ~p ***************~n", 87 [{Mod, Fun}]), 88 case eval(Mod, Fun, Config) of 89 {ok, _, _} -> 90 []; 91 Other -> 92 [Other] 93 end; 94 95 Cases when is_list(Cases) -> 96 io:format("~n*** Expand: ~p ...~n", [{Mod, Fun}]), 97 Map = fun(Case) when is_atom(Case) -> {Mod, Case}; 98 (Case) -> Case 99 end, 100 run(lists:map(Map, Cases), Config); 101 102 {conf, InitSuite, Cases, FinishSuite} when is_atom(InitSuite) andalso 103 is_list(Cases) andalso 104 is_atom(FinishSuite) -> 105 ?d("run -> conf: " 106 "~n InitSuite: ~p" 107 "~n Cases: ~p" 108 "~n FinishSuite: ~p", [InitSuite, Cases, FinishSuite]), 109 do_suite(Mod, InitSuite, Cases, FinishSuite, Config); 110 111 {req, _, SubCases} when is_list(SubCases) -> 112 ?d("run -> req: " 113 "~n SubCases: ~p", [SubCases]), 114 do_subcases(Mod, Fun, SubCases, Config, []); 115 116 {req, _, Conf} -> 117 ?d("run -> req: " 118 "~n Conf: ~p", [Conf]), 119 do_subcases(Mod, Fun, [Conf], Config, []); 120 121 {'EXIT', {undef, _}} -> 122 io:format("~n*** Undefined: ~p~n", [{Mod, Fun}]), 123 [{nyi, {Mod, Fun}, ok}]; 124 125 Error -> 126 io:format("~n*** Ignoring: ~p: ~p~n", [{Mod, Fun}, Error]), 127 [{failed, {Mod, Fun}, Error}] 128 end; 129 130run(Mod, Config) when is_atom(Mod) andalso is_list(Config) -> 131 run({Mod, all}, Config); 132 133run(Cases, Config) when is_list(Cases) andalso is_list(Config) -> 134 Errors = [run(Case, Config) || Case <- Cases], 135 lists:append(Errors); 136 137run(Bad, _Config) -> 138 [{badarg, Bad, ok}]. 139 140 141do_suite(Mod, Init, Cases, Finish, Config0) -> 142 ?d("do_suite -> entry with" 143 "~n Mod: ~p" 144 "~n Init: ~p" 145 "~n Cases: ~p" 146 "~n Finish: ~p" 147 "~n Config0: ~p", [Mod, Init, Cases, Finish, Config0]), 148 case (catch apply(Mod, Init, [Config0])) of 149 Config when is_list(Config) -> 150 io:format("~n*** Expand: ~p ...~n", [Mod]), 151 Map = fun(Case) when is_atom(Case) -> {Mod, Case}; 152 (Case) -> Case 153 end, 154 Res = run(lists:map(Map, Cases), Config), 155 (catch apply(Mod, Finish, [Config])), 156 Res; 157 158 {'EXIT', {skipped, Reason}} -> 159 io:format(" => skipping: ~p~n", [Reason]), 160 SkippedCases = 161 [{skipped, {Mod, Case}, suite_init_skipped} || Case <- Cases], 162 lists:flatten([[{skipped, {Mod, Init}, Reason}], 163 SkippedCases, 164 [{skipped, {Mod, Finish}, suite_init_skipped}]]); 165 166 Error -> 167 io:format(" => init (~p) failed: ~n~p~n", [Init, Error]), 168 InitResult = 169 [{failed, {Mod, Init}, Error}], 170 SkippedCases = 171 [{skipped, {Mod, Case}, suite_init_failed} || Case <- Cases], 172 FinResult = 173 case (catch apply(Mod, Finish, [Config0])) of 174 ok -> 175 []; 176 FinConfig when is_list(FinConfig) -> 177 []; 178 FinError -> 179 [{failed, {Mod, Finish}, FinError}] 180 end, 181 lists:flatten([InitResult, SkippedCases, FinResult]) 182 183 end. 184 185 186do_subcases(_Mod, _Fun, [], _Config, Acc) -> 187 lists:flatten(lists:reverse(Acc)); 188do_subcases(Mod, Fun, [{conf, Init, Cases, Finish}|SubCases], Config, Acc) 189 when is_atom(Init) andalso is_list(Cases) andalso is_atom(Finish) -> 190 R = case (catch apply(Mod, Init, [Config])) of 191 Conf when is_list(Conf) -> 192 io:format("~n*** Expand: ~p ...~n", [{Mod, Fun}]), 193 Map = fun(Case) when is_atom(Case) -> {Mod, Case}; 194 (Case) -> Case 195 end, 196 Res = run(lists:map(Map, Cases), Conf), 197 (catch apply(Mod, Finish, [Conf])), 198 Res; 199 200 {'EXIT', {skipped, Reason}} -> 201 io:format(" => skipping: ~p~n", [Reason]), 202 [{skipped, {Mod, Fun}, Reason}]; 203 204 Error -> 205 io:format(" => init (~p) failed: ~n~p~n", [Init, Error]), 206 (catch apply(Mod, Finish, [Config])), 207 [{failed, {Mod, Fun}, Error}] 208 end, 209 do_subcases(Mod, Fun, SubCases, Config, [R|Acc]); 210do_subcases(Mod, Fun, [SubCase|SubCases], Config, Acc) when is_atom(SubCase) -> 211 R = do_case(Mod, SubCase, Config), 212 do_subcases(Mod, Fun, SubCases,Config, [R|Acc]). 213 214 215do_case(M, F, C) -> 216 io:format("~n~n*** Eval: ~p ***************~n", [{M, F}]), 217 case eval(M, F, C) of 218 {ok, _, _} -> 219 []; 220 Other -> 221 [Other] 222 end. 223 224 225eval(Mod, Fun, Config) -> 226 Flag = process_flag(trap_exit, true), 227 global:register_name(?TEST_CASE_SUP, self()), 228 Config2 = Mod:init_per_testcase(Fun, Config), 229 Self = self(), 230 Eval = fun() -> do_eval(Self, Mod, Fun, Config2) end, 231 Pid = spawn_link(Eval), 232 R = wait_for_evaluator(Pid, Mod, Fun, Config2, []), 233 Mod:end_per_testcase(Fun, Config2), 234 global:unregister_name(?TEST_CASE_SUP), 235 process_flag(trap_exit, Flag), 236 R. 237 238wait_for_evaluator(Pid, Mod, Fun, Config, Errors) -> 239 Pre = lists:concat(["TEST CASE: ", Fun]), 240 receive 241 {'EXIT', _Watchdog, watchdog_timeout} -> 242 io:format("*** ~s WATCHDOG TIMEOUT~n", [Pre]), 243 exit(Pid, kill), 244 {failed, {Mod, Fun}, watchdog_timeout}; 245 {done, Pid, ok} when Errors =:= [] -> 246 io:format("*** ~s OK~n", [Pre]), 247 {ok, {Mod, Fun}, Errors}; 248 {done, Pid, {ok, _}} when Errors =:= [] -> 249 io:format("*** ~s OK~n", [Pre]), 250 {ok, {Mod, Fun}, Errors}; 251 {done, Pid, Fail} -> 252 io:format("*** ~s FAILED~n~p~n", [Pre, Fail]), 253 {failed, {Mod, Fun}, Fail}; 254 {'EXIT', Pid, {skipped, Reason}} -> 255 io:format("*** ~s SKIPPED~n~p~n", [Pre, Reason]), 256 {skipped, {Mod, Fun}, Errors}; 257 {'EXIT', Pid, Reason} -> 258 io:format("*** ~s CRASHED~n~p~n", [Pre, Reason]), 259 {crashed, {Mod, Fun}, [{'EXIT', Reason} | Errors]}; 260 {fail, Pid, Reason} -> 261 io:format("*** ~s FAILED~n~p~n", [Pre, Reason]), 262 wait_for_evaluator(Pid, Mod, Fun, Config, Errors ++ [Reason]) 263 end. 264 265do_eval(ReplyTo, Mod, Fun, Config) -> 266 case (catch apply(Mod, Fun, [Config])) of 267 {'EXIT', {skipped, Reason}} -> 268 ReplyTo ! {'EXIT', self(), {skipped, Reason}}; 269 Other -> 270 ReplyTo ! {done, self(), Other} 271 end, 272 unlink(ReplyTo), 273 exit(shutdown). 274 275 276display_result([]) -> 277 io:format("TEST OK~n", []); 278 279display_result(Errors) when is_list(Errors) -> 280 Nyi = [MF || {nyi, MF, _} <- Errors], 281 Skipped = [{MF, Reason} || {skipped, MF, Reason} <- Errors], 282 Crashed = [{MF, Reason} || {crashed, MF, Reason} <- Errors], 283 Failed = [{MF, Reason} || {failed, MF, Reason} <- Errors], 284 display_skipped(Skipped), 285 display_crashed(Crashed), 286 display_failed(Failed), 287 display_summery(Nyi, Skipped, Crashed, Failed). 288 289display_summery(Nyi, Skipped, Crashed, Failed) -> 290 io:format("~nTest case summery:~n", []), 291 display_summery(Nyi, "not yet implemented"), 292 display_summery(Skipped, "skipped"), 293 display_summery(Crashed, "crashed"), 294 display_summery(Failed, "failed"), 295 io:format("~n", []). 296 297display_summery([], _) -> 298 ok; 299display_summery(Res, Info) -> 300 io:format(" ~w test cases ~s~n", [length(Res), Info]). 301 302display_skipped([]) -> 303 io:format("Skipped test cases: -~n", []); 304display_skipped(Skipped) -> 305 io:format("Skipped test cases:~n", []), 306 [io:format(" ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Skipped], 307 io:format("~n", []). 308 309display_crashed([]) -> 310 io:format("Crashed test cases: -~n", []); 311display_crashed(Crashed) -> 312 io:format("Crashed test cases:~n", []), 313 [io:format(" ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Crashed], 314 io:format("~n", []). 315 316display_failed([]) -> 317 io:format("Failed test cases: -~n", []); 318display_failed(Failed) -> 319 io:format("Failed test cases:~n", []), 320 [io:format(" ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Failed], 321 io:format("~n", []). 322 323 324%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 325%% Verify that the actual result of a test case matches the exected one 326%% Returns the actual result 327%% Stores the result in the process dictionary if mismatch 328 329error(Actual, Mod, Line) -> 330 global:send(?GLOBAL_LOGGER, {failed, Mod, Line}), 331 log("<ERROR> Bad result: ~p~n", [Actual], Mod, Line), 332 case global:whereis_name(?TEST_CASE_SUP) of 333 undefined -> 334 ignore; 335 Pid -> 336 Pid ! {fail, self(), {Actual, Mod, Line}} 337 end, 338 Actual. 339 340skip(Actual, Mod, Line) -> 341 log("Skipping test case~n", [], Mod, Line), 342 exit({skipped, {Actual, Mod, Line}}). 343 344fatal_skip(Actual, Mod, Line) -> 345 error(Actual, Mod, Line), 346 exit(shutdown). 347 348 349log(Format, Args, Mod, Line) -> 350 case global:whereis_name(?GLOBAL_LOGGER) of 351 undefined -> 352 io:format(user, "~p(~p): " ++ Format, [Mod, Line] ++ Args); 353 Pid -> 354 io:format(Pid, "~p(~p): " ++ Format, [Mod, Line] ++ Args) 355 end. 356 357 358%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 359%% Test server callbacks 360 361init_per_testcase(_Case, Config) -> 362 global:register_name(?GLOBAL_LOGGER, group_leader()), 363 Config. 364 365end_per_testcase(_Case, _Config) -> 366 global:unregister_name(?GLOBAL_LOGGER), 367 ok. 368 369 370%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 371%% Internal utility functions 372 373default_config(Mod) -> 374 PrivDir0 = ?snmp_priv_dir, 375 case filename:pathtype(PrivDir0) of 376 absolute -> 377 ok; 378 _ -> 379 case file:make_dir(Mod) of 380 ok -> 381 ok; 382 {error, eexist} -> 383 ok 384 end, 385 PrivDir = filename:join(Mod, PrivDir0), 386 case file:make_dir(PrivDir) of 387 ok -> 388 ok; 389 {error, eexist} -> 390 ok; 391 Error -> 392 ?FAIL({failed_creating_subsuite_top_dir, Error}) 393 end, 394 [{priv_dir, PrivDir}] 395 end. 396 397 398%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 399 400d(F, A, L) -> 401 d(true, F, A, L). 402 %% d(get(dbg), F, A, L). 403 404d(true, F, A, L) -> 405 io:format("STS[~w] ~p " ++ F ++ "~n", [L,self()|A]); 406d(_, _, _, _) -> 407 ok. 408 409timestamp() -> 410 {Date, Time} = calendar:now_to_datetime( erlang:timestamp() ), 411 {YYYY, MM, DD} = Date, 412 {Hour, Min, Sec} = Time, 413 FormatDate = 414 io_lib:format("~.4w-~.2.0w-~.2.0w_~.2.0w.~.2.0w.~.2.0w", 415 [YYYY,MM,DD,Hour,Min,Sec]), 416 lists:flatten(FormatDate). 417 418