1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2004-2018. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20 21%%% Common Test Framework callback module. 22%%% 23%%% This module exports framework callback functions which are 24%%% called from the test_server. 25 26-module(ct_framework). 27 28-export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]). 29-export([report/2, warn/1, error_notification/4]). 30 31-export([get_log_dir/0, get_logopts/0, format_comment/1, get_html_wrapper/4]). 32 33-export([error_in_suite/1, init_per_suite/1, end_per_suite/1, 34 init_per_group/2, end_per_group/2]). 35 36-include("ct.hrl"). 37-include("ct_event.hrl"). 38-include("ct_util.hrl"). 39 40-define(val(Key, List), proplists:get_value(Key, List)). 41-define(val(Key, List, Def), proplists:get_value(Key, List, Def)). 42-define(rev(L), lists:reverse(L)). 43 44%%%----------------------------------------------------------------- 45%%% -spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} | 46%%% {skip,Reason} | {auto_skip,Reason} 47%%% Mod = atom() 48%%% Func = atom() 49%%% Args = list() 50%%% NewArgs = list() 51%%% Reason = term() 52%%% 53%%% Test server framework callback, called by the test_server 54%%% when a new test case is started. 55init_tc(_,{end_per_testcase_not_run,_},[Config]) -> 56 %% Testcase is completed (skipped or failed), but end_per_testcase 57 %% is not run - don't call pre-hook. 58 {ok,[Config]}; 59init_tc(Mod,EPTC={end_per_testcase,_},[Config]) -> 60 %% in case Mod == ct_framework, lookup the suite name 61 Suite = get_suite_name(Mod, Config), 62 case ct_hooks:init_tc(Suite,EPTC,Config) of 63 NewConfig when is_list(NewConfig) -> 64 {ok,[NewConfig]}; 65 Other-> 66 Other 67 end; 68 69init_tc(Mod,Func0,Args) -> 70 %% in case Mod == ct_framework, lookup the suite name 71 Suite = get_suite_name(Mod, Args), 72 {Func,HookFunc} = case Func0 of 73 {init_per_testcase,F} -> {F,Func0}; 74 _ -> {Func0,Func0} 75 end, 76 77 %% check if previous testcase was interpreted and has left 78 %% a "dead" trace window behind - if so, kill it 79 case ct_util:get_testdata(interpret) of 80 {What,kill,{TCPid,AttPid}} -> 81 ct_util:kill_attached(TCPid,AttPid), 82 ct_util:set_testdata({interpret,{What,kill,{undefined,undefined}}}); 83 _ -> 84 ok 85 end, 86 87 case Func=/=end_per_suite 88 andalso Func=/=end_per_group 89 andalso ct_util:get_testdata(skip_rest) of 90 true -> 91 initialize(false,Mod,Func,Args), 92 {auto_skip,"Repeated test stopped by force_stop option"}; 93 _ -> 94 case ct_util:get_testdata(curr_tc) of 95 {Suite,{suite0_failed,{require,Reason}}} -> 96 initialize(false,Mod,Func,Args), 97 {auto_skip,{require_failed_in_suite0,Reason}}; 98 {Suite,{suite0_failed,_}=Failure} -> 99 initialize(false,Mod,Func,Args), 100 {fail,Failure}; 101 _ -> 102 ct_util:update_testdata(curr_tc, 103 fun(undefined) -> 104 [{Suite,Func}]; 105 (Running) -> 106 [{Suite,Func}|Running] 107 end, [create]), 108 case ct_util:read_suite_data({seq,Suite,Func}) of 109 undefined -> 110 init_tc1(Mod,Suite,Func,HookFunc,Args); 111 Seq when is_atom(Seq) -> 112 case ct_util:read_suite_data({seq,Suite,Seq}) of 113 [Func|TCs] -> % this is the 1st case in Seq 114 %% make sure no cases in this seq are 115 %% marked as failed from an earlier execution 116 %% in the same suite 117 lists:foreach( 118 fun(TC) -> 119 ct_util:save_suite_data( 120 {seq,Suite,TC}, 121 Seq) 122 end, TCs); 123 _ -> 124 ok 125 end, 126 init_tc1(Mod,Suite,Func,HookFunc,Args); 127 {failed,Seq,BadFunc} -> 128 initialize(false,Mod,Func,Args), 129 {auto_skip,{sequence_failed,Seq,BadFunc}} 130 end 131 end 132 end. 133 134init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) -> 135 initialize(false,?MODULE,error_in_suite), 136 _ = ct_suite_init(?MODULE,error_in_suite,[],Config0), 137 case ?val(error,Config0) of 138 undefined -> 139 {fail,"unknown_error_in_suite"}; 140 Reason -> 141 {fail,Reason} 142 end; 143 144init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) -> 145 Config1 = 146 case ct_util:read_suite_data(last_saved_config) of 147 {{Suite,LastFunc},SavedConfig} -> % last testcase 148 [{saved_config,{LastFunc,SavedConfig}} | 149 lists:keydelete(saved_config,1,Config0)]; 150 {{LastSuite,InitOrEnd}, 151 SavedConfig} when InitOrEnd == init_per_suite ; 152 InitOrEnd == end_per_suite -> 153 %% last suite 154 [{saved_config,{LastSuite,SavedConfig}} | 155 lists:keydelete(saved_config,1,Config0)]; 156 undefined -> 157 lists:keydelete(saved_config,1,Config0) 158 end, 159 ct_util:delete_suite_data(last_saved_config), 160 Config = lists:keydelete(watchdog,1,Config1), 161 162 if Func == init_per_suite -> 163 %% delete all default values used in previous suite 164 ct_config:delete_default_config(suite), 165 %% release all name -> key bindings (once per suite) 166 ct_config:release_allocated(); 167 Func /= init_per_suite -> 168 ok 169 end, 170 171 GroupPath = ?val(tc_group_path, Config, []), 172 AllGroups = [?val(tc_group_properties, Config, []) | GroupPath], 173 174 %% clear all config data default values set by previous 175 %% testcase info function (these should only survive the 176 %% testcase, not the whole suite) 177 FuncSpec = group_or_func(Func,Config0), 178 HookFunc1 = 179 if is_tuple(FuncSpec) -> % group 180 FuncSpec; 181 true -> 182 ct_config:delete_default_config(testcase), 183 HookFunc 184 end, 185 case add_defaults(Mod,Func,AllGroups) of 186 Error = {suite0_failed,_} -> 187 initialize(false,Mod,FuncSpec), 188 ct_util:set_testdata({curr_tc,{Suite,Error}}), 189 {error,Error}; 190 Error = {group0_failed,_} -> 191 initialize(false,Mod,FuncSpec), 192 {auto_skip,Error}; 193 Error = {testcase0_failed,_} -> 194 initialize(false,Mod,FuncSpec), 195 {auto_skip,Error}; 196 {SuiteInfo,MergeResult} -> 197 case MergeResult of 198 {error,Reason} -> 199 initialize(false,Mod,FuncSpec), 200 {fail,Reason}; 201 _ -> 202 init_tc2(Mod,Suite,Func,HookFunc1, 203 SuiteInfo,MergeResult,Config) 204 end 205 end; 206 207init_tc1(_Mod,_Suite,_Func,_HookFunc,Args) -> 208 {ok,Args}. 209 210init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) -> 211 %% timetrap must be handled before require 212 MergedInfo = timetrap_first(MergeResult, [], []), 213 %% tell logger to use specified style sheet 214 _ = case lists:keysearch(stylesheet,1,MergeResult++Config) of 215 {value,{stylesheet,SSFile}} -> 216 ct_logs:set_stylesheet(Func,add_data_dir(SSFile,Config)); 217 _ -> 218 case ct_util:get_testdata(stylesheet) of 219 undefined -> 220 ct_logs:clear_stylesheet(Func); 221 SSFile -> 222 ct_logs:set_stylesheet(Func,SSFile) 223 end 224 end, 225 %% suppress output for connections (Conns is a 226 %% list of {Type,Bool} tuples, e.g. {telnet,true}), 227 case ct_util:get_overridden_silenced_connections() of 228 undefined -> 229 case lists:keysearch(silent_connections,1,MergeResult++Config) of 230 {value,{silent_connections,Conns}} -> 231 ct_util:silence_connections(Conns); 232 _ -> 233 ok 234 end; 235 Conns -> 236 ct_util:silence_connections(Conns) 237 end, 238 FuncSpec = group_or_func(Func,Config), 239 initialize((Func==init_per_suite),Mod,FuncSpec), 240 241 case catch configure(MergedInfo,MergedInfo,SuiteInfo, 242 FuncSpec,[],Config) of 243 {suite0_failed,Reason} -> 244 ct_util:set_testdata({curr_tc,{Mod,{suite0_failed, 245 {require,Reason}}}}), 246 {auto_skip,{require_failed_in_suite0,Reason}}; 247 {error,Reason} -> 248 {auto_skip,{require_failed,Reason}}; 249 {'EXIT',Reason} -> 250 {fail,Reason}; 251 {ok,PostInitHook,Config1} -> 252 case get('$test_server_framework_test') of 253 undefined -> 254 ct_suite_init(Suite,HookFunc,PostInitHook,Config1); 255 Fun -> 256 PostInitHookResult = do_post_init_hook(PostInitHook, 257 Config1), 258 case Fun(init_tc, [PostInitHookResult ++ Config1]) of 259 NewConfig when is_list(NewConfig) -> 260 {ok,NewConfig}; 261 Else -> 262 Else 263 end 264 end 265 end. 266 267initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) -> 268 initialize(RefreshLogs,Mod,group_or_func(Func,Config)); 269initialize(RefreshLogs,Mod,Func,_) -> 270 initialize(RefreshLogs,Mod,Func). 271 272initialize(RefreshLogs,Mod,FuncSpec) -> 273 ct_logs:init_tc(RefreshLogs), 274 ct_event:notify(#event{name=tc_start, 275 node=node(), 276 data={Mod,FuncSpec}}). 277 278 279ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) -> 280 case ct_hooks:init_tc(Suite,HookFunc,Config) of 281 NewConfig when is_list(NewConfig) -> 282 PostInitHookResult = do_post_init_hook(PostInitHook,NewConfig), 283 {ok, [PostInitHookResult ++ NewConfig]}; 284 Else -> 285 Else 286 end. 287 288do_post_init_hook(PostInitHook,Config) -> 289 lists:flatmap(fun({Tag,Fun}) -> 290 case lists:keysearch(Tag,1,Config) of 291 {value,_} -> 292 []; 293 false -> 294 case Fun() of 295 {error,_} -> []; 296 Result -> [{Tag,Result}] 297 end 298 end 299 end, PostInitHook). 300 301add_defaults(Mod,Func, GroupPath) -> 302 Suite = get_suite_name(Mod, GroupPath), 303 case (catch Suite:suite()) of 304 {'EXIT',{undef,_}} -> 305 SuiteInfo = merge_with_suite_defaults(Suite,[]), 306 SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks], 307 case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of 308 Error = {group0_failed,_} -> Error; 309 Error = {testcase0_failed,_} -> Error; 310 Error = {error,_} -> {SuiteInfo,Error}; 311 MergedInfo -> {SuiteInfo,MergedInfo} 312 end; 313 {'EXIT',Reason} -> 314 ErrStr = io_lib:format("~n*** ERROR *** " 315 "~w:suite/0 failed: ~tp~n", 316 [Suite,Reason]), 317 io:format(ErrStr, []), 318 io:format(?def_gl, ErrStr, []), 319 {suite0_failed,{exited,Reason}}; 320 SuiteInfo when is_list(SuiteInfo) -> 321 case lists:all(fun(E) when is_tuple(E) -> true; 322 (_) -> false 323 end, SuiteInfo) of 324 true -> 325 SuiteInfo1 = merge_with_suite_defaults(Suite, SuiteInfo), 326 SuiteInfoNoCTH = [I || I <- SuiteInfo1, 327 element(1,I) =/= ct_hooks], 328 case add_defaults1(Mod,Func, GroupPath, 329 SuiteInfoNoCTH) of 330 Error = {group0_failed,_} -> Error; 331 Error = {testcase0_failed,_} -> Error; 332 Error = {error,_} -> {SuiteInfo1,Error}; 333 MergedInfo -> {SuiteInfo1,MergedInfo} 334 end; 335 false -> 336 ErrStr = io_lib:format("~n*** ERROR *** " 337 "Invalid return value from " 338 "~w:suite/0: ~tp~n", 339 [Suite,SuiteInfo]), 340 io:format(ErrStr, []), 341 io:format(?def_gl, ErrStr, []), 342 {suite0_failed,bad_return_value} 343 end; 344 SuiteInfo -> 345 ErrStr = io_lib:format("~n*** ERROR *** " 346 "Invalid return value from " 347 "~w:suite/0: ~tp~n", [Suite,SuiteInfo]), 348 io:format(ErrStr, []), 349 io:format(?def_gl, ErrStr, []), 350 {suite0_failed,bad_return_value} 351 end. 352 353add_defaults1(Mod,Func, GroupPath, SuiteInfo) -> 354 Suite = get_suite_name(Mod, GroupPath), 355 %% GroupPathInfo (for subgroup on level X) = 356 %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo] 357 GroupPathInfo = 358 lists:map(fun(GroupProps) -> 359 case ?val(name, GroupProps) of 360 undefined -> 361 []; 362 Name -> 363 case catch Suite:group(Name) of 364 GrInfo when is_list(GrInfo) -> GrInfo; 365 {'EXIT',{undef,_}} -> []; 366 BadGr0 -> {error,BadGr0,Name} 367 end 368 end 369 end, GroupPath), 370 case lists:keysearch(error, 1, GroupPathInfo) of 371 {value,{error,BadGr0Val,GrName}} -> 372 Gr0ErrStr = io_lib:format("~n*** ERROR *** " 373 "Invalid return value from " 374 "~w:group(~tw): ~tp~n", 375 [Mod,GrName,BadGr0Val]), 376 io:format(Gr0ErrStr, []), 377 io:format(?def_gl, Gr0ErrStr, []), 378 {group0_failed,bad_return_value}; 379 _ -> 380 Args = if Func == init_per_group ; Func == end_per_group -> 381 [?val(name, hd(GroupPath))]; 382 true -> 383 [] 384 end, 385 TestCaseInfo = 386 case catch apply(Mod,Func,Args) of 387 TCInfo when is_list(TCInfo) -> TCInfo; 388 {'EXIT',{undef,_}} -> []; 389 BadTC0 -> {error,BadTC0} 390 end, 391 392 case TestCaseInfo of 393 {error,BadTC0Val} -> 394 TC0ErrStr = io_lib:format("~n*** ERROR *** " 395 "Invalid return value from " 396 "~w:~tw/0: ~tp~n", 397 [Mod,Func,BadTC0Val]), 398 io:format(TC0ErrStr, []), 399 io:format(?def_gl, TC0ErrStr, []), 400 {testcase0_failed,bad_return_value}; 401 _ -> 402 %% let test case info (also for all config funcs) override 403 %% group info, and lower level group info override higher 404 %% level info 405 TCAndGroupInfo = 406 [TestCaseInfo | remove_info_in_prev(TestCaseInfo, 407 GroupPathInfo)], 408 %% find and save require terms found in suite info 409 SuiteReqs = 410 [SDDef || SDDef <- SuiteInfo, 411 ((require == element(1,SDDef)) 412 or (default_config == element(1,SDDef)))], 413 case check_for_clashes(TestCaseInfo, GroupPathInfo, 414 SuiteReqs) of 415 [] -> 416 add_defaults2(Mod,Func, TCAndGroupInfo, 417 SuiteInfo,SuiteReqs); 418 Clashes -> 419 {error,{config_name_already_in_use,Clashes}} 420 end 421 end 422 end. 423 424get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] -> 425 get_suite_name(?MODULE, Cfg); 426 427get_suite_name(?MODULE, Cfg) when is_list(Cfg), Cfg /= [] -> 428 case ?val(tc_group_properties, Cfg) of 429 undefined -> 430 case ?val(suite, Cfg) of 431 undefined -> ?MODULE; 432 Suite -> Suite 433 end; 434 GrProps -> 435 case ?val(suite, GrProps) of 436 undefined -> ?MODULE; 437 Suite -> Suite 438 end 439 end; 440get_suite_name(Mod, _) -> 441 Mod. 442 443%% Check that alias names are not already in use 444check_for_clashes(TCInfo, [CurrGrInfo|Path], SuiteInfo) -> 445 ReqNames = fun(Info) -> [element(2,R) || R <- Info, 446 size(R) == 3, 447 require == element(1,R)] 448 end, 449 ExistingNames = lists:flatten([ReqNames(L) || L <- [SuiteInfo|Path]]), 450 CurrGrReqNs = ReqNames(CurrGrInfo), 451 GrClashes = [Name || Name <- CurrGrReqNs, 452 true == lists:member(Name, ExistingNames)], 453 AllReqNs = CurrGrReqNs ++ ExistingNames, 454 TCClashes = [Name || Name <- ReqNames(TCInfo), 455 true == lists:member(Name, AllReqNs)], 456 TCClashes ++ GrClashes. 457 458%% Delete the info terms in Terms from all following info lists 459remove_info_in_prev(Terms, [[] | Rest]) -> 460 [[] | remove_info_in_prev(Terms, Rest)]; 461remove_info_in_prev(Terms, [Info | Rest]) -> 462 UniqueInInfo = [U || U <- Info, 463 ((timetrap == element(1,U)) and 464 (not lists:keymember(timetrap,1,Terms))) or 465 ((require == element(1,U)) and 466 (not lists:member(U,Terms))) or 467 ((default_config == element(1,U)) and 468 (not keysmember([default_config,1, 469 element(2,U),2], Terms)))], 470 OtherTermsInInfo = [T || T <- Info, 471 timetrap /= element(1,T), 472 require /= element(1,T), 473 default_config /= element(1,T), 474 false == lists:keymember(element(1,T),1, 475 Terms)], 476 KeptInfo = UniqueInInfo ++ OtherTermsInInfo, 477 [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)]; 478remove_info_in_prev(_, []) -> 479 []. 480 481keysmember([Key,Pos|Next], List) -> 482 case [Elem || Elem <- List, Key == element(Pos,Elem)] of 483 [] -> false; 484 Found -> keysmember(Next, Found) 485 end; 486keysmember([], _) -> true. 487 488 489add_defaults2(_Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs) -> 490 Info = lists:flatten([IPSInfo, SuiteReqs]), 491 lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]); 492 493add_defaults2(_Mod,init_per_group, IPGAndGroupInfo, SuiteInfo,SuiteReqs) -> 494 SuiteInfo1 = 495 remove_info_in_prev(lists:flatten([IPGAndGroupInfo, 496 SuiteReqs]), [SuiteInfo]), 497 %% don't require terms in prev groups (already processed) 498 case IPGAndGroupInfo of 499 [IPGInfo] -> 500 lists:flatten([IPGInfo,SuiteInfo1]); 501 [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] -> 502 PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), 503 lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1, 504 SuiteInfo1]) 505 end; 506 507add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs) -> 508 %% Include require elements from test case info and current group, 509 %% but not from previous groups or suite/0 (since we've already required 510 %% those vars). Let test case info elements override group and suite 511 %% info elements. 512 SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo, 513 SuiteReqs]), [SuiteInfo]), 514 %% don't require terms in prev groups (already processed) 515 case TCAndGroupInfo of 516 [TCInfo] -> 517 lists:flatten([TCInfo,SuiteInfo1]); 518 [TCInfo | [CurrGroupInfo | PrevGroupInfo]] -> 519 PrevGroupInfo1 = delete_require_terms(PrevGroupInfo), 520 lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1, 521 SuiteInfo1]) 522 end. 523 524delete_require_terms([Info | Prev]) -> 525 Info1 = [T || T <- Info, 526 require /= element(1,T), 527 default_config /= element(1,T)], 528 [Info1 | delete_require_terms(Prev)]; 529delete_require_terms([]) -> 530 []. 531 532merge_with_suite_defaults(Mod,SuiteInfo) -> 533 case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of 534 {value,{suite_defaults,Defaults}} -> 535 SDReqs = 536 [SDDef || SDDef <- Defaults, 537 require == element(1,SDDef), 538 false == lists:keymember(element(2,SDDef),2, 539 SuiteInfo)], 540 SuiteInfo ++ SDReqs ++ 541 [SDDef || SDDef <- Defaults, 542 require /= element(1,SDDef), 543 false == lists:keymember(element(1,SDDef),1, 544 SuiteInfo)]; 545 false -> 546 SuiteInfo 547 end. 548 549timetrap_first([Trap = {timetrap,_} | Rest],Info,Found) -> 550 timetrap_first(Rest,Info,[Trap | Found]); 551timetrap_first([Other | Rest],Info,Found) -> 552 timetrap_first(Rest,[Other | Info],Found); 553timetrap_first([],Info,[]) -> 554 [{timetrap,{minutes,30}} | ?rev(Info)]; 555timetrap_first([],Info,Found) -> 556 ?rev(Found) ++ ?rev(Info). 557 558configure([{require,Required}|Rest], 559 Info,SuiteInfo,Scope,PostInitHook,Config) -> 560 case ct:require(Required) of 561 ok -> 562 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config); 563 Error = {error,Reason} -> 564 case required_default('_UNDEF',Required,Info, 565 SuiteInfo,Scope) of 566 ok -> 567 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config); 568 _ -> 569 case lists:keymember(Required,2,SuiteInfo) of 570 true -> 571 {suite0_failed,Reason}; 572 false -> 573 Error 574 end 575 end 576 end; 577configure([{require,Name,Required}|Rest], 578 Info,SuiteInfo,Scope,PostInitHook,Config) -> 579 case ct:require(Name,Required) of 580 ok -> 581 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config); 582 Error = {error,Reason} -> 583 case required_default(Name,Required,Info,SuiteInfo,Scope) of 584 ok -> 585 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config); 586 _ -> 587 case lists:keymember(Name,2,SuiteInfo) of 588 true -> 589 {suite0_failed,Reason}; 590 false -> 591 Error 592 end 593 end 594 end; 595configure([{timetrap,off}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) -> 596 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config); 597configure([{timetrap,Time}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) -> 598 PostInitHook1 = 599 [{watchdog,fun() -> case test_server:get_timetrap_info() of 600 undefined -> 601 test_server:timetrap(Time); 602 _ -> 603 {error,already_set} 604 end 605 end} | PostInitHook], 606 configure(Rest,Info,SuiteInfo,Scope,PostInitHook1,Config); 607configure([{ct_hooks,Hook}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) -> 608 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,[{ct_hooks,Hook}|Config]); 609configure([_|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) -> 610 configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config); 611configure([],_,_,_,PostInitHook,Config) -> 612 {ok,PostInitHook,Config}. 613 614%% the require element in Info may come from suite/0 and 615%% should be scoped 'suite', or come from the group info 616%% function and be scoped 'group', or come from the testcase 617%% info function and then be scoped 'testcase' 618 619required_default(Name,Key,Info,_,init_per_suite) -> 620 try_set_default(Name,Key,Info,suite); 621required_default(Name,Key,Info,_,{init_per_group,GrName,_}) -> 622 try_set_default(Name,Key,Info,{group,GrName}); 623required_default(Name,Key,Info,_,_FuncSpec) -> 624 try_set_default(Name,Key,Info,testcase). 625 626try_set_default(Name,Key,Info,Where) -> 627 CfgElems = 628 case lists:keysearch(Name,1,Info) of 629 {value,{Name,Val}} -> 630 [Val]; 631 false -> 632 case catch [{Key,element(3,Elem)} || Elem <- Info, 633 element(1,Elem)==default_config, 634 element(2,Elem)==Key] of 635 {'EXIT',_} -> []; 636 Result -> Result 637 end 638 end, 639 case {Name,CfgElems} of 640 {_,[]} -> 641 no_default; 642 {'_UNDEF',_} -> 643 _ = [ct_config:set_default_config([CfgVal],Where) || CfgVal <- CfgElems], 644 ok; 645 _ -> 646 _ = [ct_config:set_default_config(Name,[CfgVal],Where) || CfgVal <- CfgElems], 647 ok 648 end. 649 650 651%%%----------------------------------------------------------------- 652%%% -spec end_tc(Mod,Func,Args) -> {ok,NewArgs}| {error,Reason} | 653%%% {skip,Reason} | {auto_skip,Reason} 654%%% Mod = atom() 655%%% Func = atom() 656%%% Args = list() 657%%% NewArgs = list() 658%%% Reason = term() 659%%% 660%%% Test server framework callback, called by the test_server 661%%% when a test case is finished. 662end_tc(Mod, Fun, Args) -> 663 %% Have to keep end_tc/3 for backwards compatibility issues 664 end_tc(Mod, Fun, Args, '$end_tc_dummy'). 665end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) -> 666 %% this clause gets called if CT has encountered a suite that 667 %% can't be executed 668 FinalNotify = 669 case ct_hooks:end_tc(?MODULE, error_in_suite, Args, Result, Return) of 670 '$ct_no_change' -> 671 Result; 672 HookResult -> 673 HookResult 674 end, 675 Event = #event{name=tc_done, 676 node=node(), 677 data={?MODULE,error_in_suite,tag(FinalNotify)}}, 678 ct_event:sync_notify(Event), 679 ok; 680end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) -> 681 end_tc(Mod,Func,TCPid,Result,Args,Return); 682end_tc(Mod,Func,{Result,[Args]}, Return) -> 683 end_tc(Mod,Func,self(),Result,Args,Return). 684 685end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> 686 case end_hook_func(IPTC,Return,IPTC) of 687 undefined -> ok; 688 _ -> 689 %% in case Mod == ct_framework, lookup the suite name 690 Suite = get_suite_name(Mod, Args), 691 case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of 692 '$ct_no_change' -> 693 ok; 694 HookResult -> 695 HookResult 696 end 697 end; 698 699end_tc(Mod,Func00,TCPid,Result,Args,Return) -> 700 %% in case Mod == ct_framework, lookup the suite name 701 Suite = get_suite_name(Mod, Args), 702 {OnlyCleanup,Func0} = 703 case Func00 of 704 {cleanup,F0} -> 705 {true,F0}; 706 _ -> 707 {false,Func00} 708 end, 709 {Func,FuncSpec,HookFunc} = 710 case Func0 of 711 {end_per_testcase_not_run,F} -> 712 %% Testcase is completed (skipped or failed), but 713 %% end_per_testcase is not run - don't call post-hook. 714 {F,F,undefined}; 715 {end_per_testcase,F} -> 716 {F,F,Func0}; 717 _ -> 718 FS = group_or_func(Func0,Args), 719 HF = end_hook_func(Func0,Return,FS), 720 {Func0,FS,HF} 721 end, 722 723 test_server:timetrap_cancel(), 724 725 %% save the testcase process pid so that it can be used 726 %% to look up the attached trace window later 727 case ct_util:get_testdata(interpret) of 728 {What,kill,_} -> 729 AttPid = ct_util:get_attached(self()), 730 ct_util:set_testdata({interpret,{What,kill,{self(),AttPid}}}); 731 _ -> 732 ok 733 end, 734 if Func == end_per_group; Func == end_per_suite -> 735 %% clean up any saved comments 736 ct_util:match_delete_testdata({comment,'_'}); 737 true -> 738 %% attemp to delete any saved comment for this TC 739 case process_info(TCPid, group_leader) of 740 {group_leader,TCGL} -> 741 ct_util:delete_testdata({comment,TCGL}); 742 _ -> 743 ok 744 end 745 end, 746 ct_util:delete_suite_data(last_saved_config), 747 748 {Result1,FinalNotify} = 749 case HookFunc of 750 undefined -> 751 {ok,Result}; 752 _ when OnlyCleanup -> 753 {ok,Result}; 754 _ -> 755 case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of 756 '$ct_no_change' -> 757 {ok,Result}; 758 HookResult -> 759 {HookResult,HookResult} 760 end 761 end, 762 FinalResult = 763 case get('$test_server_framework_test') of 764 _ when OnlyCleanup -> 765 Result1; 766 undefined -> 767 %% send sync notification so that event handlers may print 768 %% in the log file before it gets closed 769 Event = #event{name=tc_done, 770 node=node(), 771 data={Mod,FuncSpec, 772 tag(FinalNotify)}}, 773 ct_event:sync_notify(Event), 774 Result1; 775 Fun -> 776 %% send sync notification so that event handlers may print 777 %% in the log file before it gets closed 778 Event = #event{name=tc_done, 779 node=node(), 780 data={Mod,FuncSpec, 781 tag({'$test_server_framework_test', 782 FinalNotify})}}, 783 ct_event:sync_notify(Event), 784 Fun(end_tc, Return) 785 end, 786 787 case FuncSpec of 788 {_,GroupName,_Props} -> 789 if Func == end_per_group -> 790 ct_config:delete_default_config({group,GroupName}); 791 true -> ok 792 end, 793 case lists:keysearch(save_config,1,Args) of 794 {value,{save_config,SaveConfig}} -> 795 ct_util:save_suite_data(last_saved_config, 796 {Suite,{group,GroupName}}, 797 SaveConfig); 798 false -> 799 ok 800 end; 801 _ -> 802 case lists:keysearch(save_config,1,Args) of 803 {value,{save_config,SaveConfig}} -> 804 ct_util:save_suite_data(last_saved_config, 805 {Suite,Func},SaveConfig); 806 false -> 807 ok 808 end 809 end, 810 811 ct_util:reset_silent_connections(), 812 813 %% reset the curr_tc state, or delete this TC from the list of 814 %% executing cases (if in a parallel group) 815 ClearCurrTC = fun(Running = [_,_|_]) -> 816 lists:keydelete(Func,2,Running); 817 ({_,{suite0_failed,_}}) -> 818 undefined; 819 ([{_,CurrTC}]) when CurrTC == Func -> 820 undefined; 821 (undefined) -> 822 undefined; 823 (Unexpected) -> 824 {error,{reset_curr_tc,{Mod,Func},Unexpected}} 825 end, 826 case ct_util:update_testdata(curr_tc, ClearCurrTC) of 827 {error,_} = ClearError -> 828 exit(ClearError); 829 _ -> 830 ok 831 end, 832 833 case FinalResult of 834 {auto_skip,{sequence_failed,_,_}} -> 835 %% ct_logs:init_tc is never called for a skipped test case 836 %% in a failing sequence, so neither should end_tc 837 ok; 838 _ -> 839 case ct_logs:end_tc(TCPid) of 840 {error,Reason} -> 841 exit({error,{logger,Reason}}); 842 _ -> 843 ok 844 end 845 end, 846 case Func of 847 end_per_suite -> 848 ct_util:match_delete_suite_data({seq,Suite,'_'}); 849 _ -> 850 ok 851 end, 852 FinalResult. 853 854%% This is to make sure that no post_init_per_* is ever called if the 855%% corresponding pre_init_per_* was not called. 856%% The skip or fail reasons are those that can be returned from 857%% init_tc above in situations where we never came to call 858%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call 859%% ct_hooks:init_tc for init_per_suite, and thus we must not call 860%% ct_hooks:end_tc for init_per_suite either. 861end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) -> 862 undefined; 863end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) -> 864 undefined; 865end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) -> 866 undefined; 867end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_) 868 when InfoFuncError==testcase0_failed; 869 InfoFuncError==require_failed -> 870 undefined; 871end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_) 872 when InfoFuncError==group0_failed; 873 InfoFuncError==require_failed -> 874 undefined; 875end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) -> 876 undefined; 877end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) -> 878 undefined; 879end_hook_func(_,_,Default) -> 880 Default. 881 882%% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | 883%% {testcase_aborted,Reason} | testcase_aborted_or_killed | 884%% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} | 885%% {user_timetrap_error,Reason} | 886%% Other (ignored return value, e.g. 'ok') 887tag({'$test_server_framework_test',Result}) -> 888 case tag(Result) of 889 ok -> Result; 890 Failure -> Failure 891 end; 892tag({skipped,Reason={failed,{_,init_per_testcase,_}}}) -> 893 {auto_skipped,Reason}; 894tag({STag,Reason}) when STag == skip; STag == skipped -> 895 case Reason of 896 {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason}; 897 _ -> {skipped,Reason} 898 end; 899tag({auto_skip,Reason}) -> 900 {auto_skipped,Reason}; 901tag({fail,Reason}) -> 902 {failed,{error,Reason}}; 903tag(Failed = {failed,_Reason}) -> 904 Failed; 905tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; 906 ETag == timetrap_timeout; 907 ETag == testcase_aborted -> 908 {failed,E}; 909tag(E = testcase_aborted_or_killed) -> 910 {failed,E}; 911tag(UserTimetrap = {user_timetrap_error,_Reason}) -> 912 UserTimetrap; 913tag(_Other) -> 914 ok. 915 916%%%----------------------------------------------------------------- 917%%% -spec error_notification(Mod,Func,Args,Error) -> ok 918%%% Mod = atom() 919%%% Func = atom() 920%%% Args = list() 921%%% Error = term() 922%%% 923%%% This function is called as the result of testcase 924%%% Func in suite Mod crashing. 925%%% Error specifies the reason for failing. 926error_notification(Mod,Func,_Args,{Error,Loc}) -> 927 ErrorSpec = case Error of 928 {What={_E,_R},Trace} when is_list(Trace) -> 929 What; 930 What -> 931 What 932 end, 933 ErrorStr = case ErrorSpec of 934 {badmatch,Descr} -> 935 Descr1 = io_lib:format("~tP",[Descr,10]), 936 DescrLength = string:length(Descr1), 937 if DescrLength > 50 -> 938 Descr2 = string:slice(Descr1,0,50), 939 io_lib:format("{badmatch,~ts...}",[Descr2]); 940 true -> 941 io_lib:format("{badmatch,~ts}",[Descr1]) 942 end; 943 {test_case_failed,Reason} -> 944 case (catch io_lib:format("{test_case_failed,~ts}", [Reason])) of 945 {'EXIT',_} -> 946 io_lib:format("{test_case_failed,~tp}", [Reason]); 947 Result -> Result 948 end; 949 {'EXIT',_Reason} = EXIT -> 950 io_lib:format("~tP", [EXIT,5]); 951 {Spec,_Reason} when is_atom(Spec) -> 952 io_lib:format("~tw", [Spec]); 953 Other -> 954 io_lib:format("~tP", [Other,5]) 955 end, 956 ErrorHtml = 957 "<font color=\"brown\">" ++ ct_logs:escape_chars(ErrorStr) ++ "</font>", 958 case {Mod,Error} of 959 %% some notifications come from the main test_server process 960 %% and for these cases the existing comment may not be modified 961 {_,{timetrap_timeout,_TVal}} -> 962 ok; 963 {_,{testcase_aborted,_Info}} -> 964 ok; 965 {_,testcase_aborted_or_killed} -> 966 ok; 967 {undefined,_OtherError} -> 968 ok; 969 _ -> 970 %% this notification comes from the test case process, so 971 %% we can add error info to comment with test_server:comment/1 972 case ct_util:get_testdata({comment,group_leader()}) of 973 undefined -> 974 test_server:comment(ErrorHtml); 975 Comment -> 976 CommentHtml = 977 "<font color=\"green\">" ++ "(" ++ "</font>" 978 ++ Comment ++ 979 "<font color=\"green\">" ++ ")" ++ "</font>", 980 Str = io_lib:format("~ts ~ts", [ErrorHtml,CommentHtml]), 981 test_server:comment(Str) 982 end 983 end, 984 985 PrintError = fun(ErrorFormat, ErrorArgs) -> 986 Div = "~n- - - - - - - - - - - - - - - - - - - " 987 "- - - - - - - - - - - - - - - - - - - - -~n", 988 ErrorStr2 = io_lib:format(ErrorFormat, ErrorArgs), 989 io:format(?def_gl, lists:concat([Div,ErrorStr2,Div,"~n"]), 990 []), 991 Link = 992 "\n\n<a href=\"#end\">" 993 "Full error description and stacktrace" 994 "</a>", 995 ErrorHtml2 = ct_logs:escape_chars(ErrorStr2), 996 ct_logs:tc_log(ct_error_notify, 997 ?MAX_IMPORTANCE, 998 "CT Error Notification", 999 ErrorHtml2++Link, [], []) 1000 end, 1001 case Loc of 1002 [{?MODULE,error_in_suite}] -> 1003 PrintError("Error in suite detected: ~ts", [ErrorStr]); 1004 1005 R when R == unknown; R == undefined -> 1006 PrintError("Error detected: ~ts", [ErrorStr]); 1007 1008 %% if a function specified by all/0 does not exist, we 1009 %% pick up undef here 1010 [{LastMod,LastFunc}|_] when ErrorStr == "undef" -> 1011 PrintError("~w:~tw could not be executed~nReason: ~ts", 1012 [LastMod,LastFunc,ErrorStr]); 1013 1014 [{LastMod,LastFunc}|_] -> 1015 PrintError("~w:~tw failed~nReason: ~ts", [LastMod,LastFunc,ErrorStr]); 1016 1017 [{LastMod,LastFunc,LastLine}|_] -> 1018 %% print error to console, we are only 1019 %% interested in the last executed expression 1020 PrintError("~w:~tw failed on line ~w~nReason: ~ts", 1021 [LastMod,LastFunc,LastLine,ErrorStr]), 1022 1023 case ct_util:read_suite_data({seq,Mod,Func}) of 1024 undefined -> 1025 ok; 1026 Seq -> 1027 SeqTCs = ct_util:read_suite_data({seq,Mod,Seq}), 1028 mark_as_failed(Seq,Mod,Func,SeqTCs) 1029 end 1030 end, 1031 ok. 1032 1033%% cases in seq that have already run 1034mark_as_failed(Seq,Mod,Func,[Func|TCs]) -> 1035 mark_as_failed1(Seq,Mod,Func,TCs); 1036mark_as_failed(Seq,Mod,Func,[_TC|TCs]) -> 1037 mark_as_failed(Seq,Mod,Func,TCs); 1038mark_as_failed(_,_,_,[]) -> 1039 ok; 1040mark_as_failed(_,_,_,undefined) -> 1041 ok. 1042 1043%% mark rest of cases in seq to be skipped 1044mark_as_failed1(Seq,Mod,Func,[TC|TCs]) -> 1045 ct_util:save_suite_data({seq,Mod,TC},{failed,Seq,Func}), 1046 mark_as_failed1(Seq,Mod,Func,TCs); 1047mark_as_failed1(_,_,_,[]) -> 1048 ok. 1049 1050group_or_func(Func, Config) when Func == init_per_group; 1051 Func == end_per_group -> 1052 case ?val(tc_group_properties, Config) of 1053 undefined -> 1054 {Func,unknown,[]}; 1055 GrProps -> 1056 GrName = ?val(name,GrProps), 1057 {Func,GrName,proplists:delete(name,GrProps)} 1058 end; 1059group_or_func(Func, _Config) -> 1060 Func. 1061 1062%%%----------------------------------------------------------------- 1063%%% -spec get_suite(Mod, Func) -> Tests 1064%%% 1065%%% Called from test_server for every suite (Func==all) 1066%%% and every test case. If the former, all test cases in the suite 1067%%% should be returned. 1068 1069get_suite(Mod, all) -> 1070 case safe_apply_groups_0(Mod,{ok,[]}) of 1071 {ok,GroupDefs} -> 1072 try ct_groups:find_groups(Mod, all, all, GroupDefs) of 1073 ConfTests when is_list(ConfTests) -> 1074 get_all(Mod, ConfTests) 1075 catch 1076 throw:{error,Error} -> 1077 [{?MODULE,error_in_suite,[[{error,Error}]]}]; 1078 _:Error:S -> 1079 [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] 1080 end; 1081 {error,{bad_return,_Bad}} -> 1082 E = "Bad return value from "++atom_to_list(Mod)++":groups/0", 1083 [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]; 1084 {error,{bad_hook_return,Bad}} -> 1085 E = "Bad return value from post_groups/2 hook function", 1086 [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}]; 1087 {error,{failed,ExitReason}} -> 1088 case ct_util:get_testdata({error_in_suite,Mod}) of 1089 undefined -> 1090 ErrStr = io_lib:format("~n*** ERROR *** " 1091 "~w:groups/0 failed: ~p~n", 1092 [Mod,ExitReason]), 1093 io:format(?def_gl, ErrStr, []), 1094 %% save the error info so it doesn't get printed twice 1095 ct_util:set_testdata_async({{error_in_suite,Mod}, 1096 ExitReason}); 1097 _ExitReason -> 1098 ct_util:delete_testdata({error_in_suite,Mod}) 1099 end, 1100 Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"), 1101 [{?MODULE,error_in_suite,[[{error,Reason}]]}]; 1102 {error,What} -> 1103 [{?MODULE,error_in_suite,[[{error,What}]]}] 1104 end; 1105 1106%%!============================================================ 1107%%! Note: The handling of sequences in get_suite/2 and get_all/2 1108%%! is deprecated and should be removed at some point... 1109%%!============================================================ 1110 1111%% group 1112get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> 1113 case safe_apply_groups_0(Mod,{ok,[Group]}) of 1114 {ok,GroupDefs} -> 1115 Name = ?val(name, Props), 1116 try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of 1117 [] -> 1118 []; 1119 ConfTests when is_list(ConfTests) -> 1120 case lists:member(skipped, Props) of 1121 true -> 1122 %% a *subgroup* specified *only* as skipped (and not 1123 %% as an explicit test) should not be returned, or 1124 %% init/end functions for top groups will be executed 1125 try ?val(name, element(2, hd(ConfTests))) of 1126 Name -> % top group 1127 ct_groups:delete_subs(ConfTests, ConfTests); 1128 _ -> [] 1129 catch 1130 _:_ -> [] 1131 end; 1132 false -> 1133 ConfTests1 = ct_groups:delete_subs(ConfTests, 1134 ConfTests), 1135 case ?val(override, Props) of 1136 undefined -> 1137 ConfTests1; 1138 [] -> 1139 ConfTests1; 1140 ORSpec -> 1141 ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; 1142 true -> ORSpec end, 1143 ct_groups:search_and_override(ConfTests1, 1144 ORSpec1, Mod) 1145 end 1146 end 1147 catch 1148 throw:{error,Error} -> 1149 [{?MODULE,error_in_suite,[[{error,Error}]]}]; 1150 _:Error:S -> 1151 [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] 1152 end; 1153 {error,{bad_return,_Bad}} -> 1154 E = "Bad return value from "++atom_to_list(Mod)++":groups/0", 1155 [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]; 1156 {error,{bad_hook_return,Bad}} -> 1157 E = "Bad return value from post_groups/2 hook function", 1158 [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}]; 1159 {error,{failed,ExitReason}} -> 1160 case ct_util:get_testdata({error_in_suite,Mod}) of 1161 undefined -> 1162 ErrStr = io_lib:format("~n*** ERROR *** " 1163 "~w:groups/0 failed: ~p~n", 1164 [Mod,ExitReason]), 1165 io:format(?def_gl, ErrStr, []), 1166 %% save the error info so it doesn't get printed twice 1167 ct_util:set_testdata_async({{error_in_suite,Mod}, 1168 ExitReason}); 1169 _ExitReason -> 1170 ct_util:delete_testdata({error_in_suite,Mod}) 1171 end, 1172 Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"), 1173 [{?MODULE,error_in_suite,[[{error,Reason}]]}]; 1174 {error,What} -> 1175 [{?MODULE,error_in_suite,[[{error,What}]]}] 1176 end; 1177 1178%% testcase 1179get_suite(Mod, Name) -> 1180 get_seq(Mod, Name). 1181 1182%%%----------------------------------------------------------------- 1183 1184get_all_cases(Suite) -> 1185 case get_suite(Suite, all) of 1186 [{?MODULE,error_in_suite,[[{error,_}=Error]]}] -> 1187 Error; 1188 [{?MODULE,error_in_suite,[[Error]]}] -> 1189 {error,Error}; 1190 Tests -> 1191 Cases = get_all_cases1(Suite, Tests), 1192 ?rev(lists:foldl(fun(TC, TCs) -> 1193 case lists:member(TC, TCs) of 1194 true -> TCs; 1195 false -> [TC | TCs] 1196 end 1197 end, [], Cases)) 1198 end. 1199 1200get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) -> 1201 get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests); 1202 1203get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) -> 1204 [{Suite,Test} | get_all_cases1(Suite, Tests)]; 1205 1206get_all_cases1(Suite, [Test | Tests]) -> 1207 [Test | get_all_cases1(Suite, Tests)]; 1208 1209get_all_cases1(_, []) -> 1210 []. 1211 1212%%%----------------------------------------------------------------- 1213 1214get_all(Mod, ConfTests) -> 1215 case safe_apply_all_0(Mod) of 1216 {ok,AllTCs} -> 1217 %% expand group references using ConfTests 1218 try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of 1219 {error,_} = Error -> 1220 [{?MODULE,error_in_suite,[[Error]]}]; 1221 Tests0 -> 1222 Tests = ct_groups:delete_subs(Tests0, Tests0), 1223 expand_tests(Mod, Tests) 1224 catch 1225 throw:{error,Error} -> 1226 [{?MODULE,error_in_suite,[[{error,Error}]]}]; 1227 _:Error:S -> 1228 [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] 1229 end; 1230 Skip = {skip,_Reason} -> 1231 Skip; 1232 {error,undef} -> 1233 Reason = 1234 case code:which(Mod) of 1235 non_existing -> 1236 list_to_atom( 1237 atom_to_list(Mod)++ 1238 " cannot be compiled or loaded"); 1239 _ -> 1240 list_to_atom( 1241 atom_to_list(Mod)++":all/0 is missing") 1242 end, 1243 %% this makes test_server call error_in_suite as first 1244 %% (and only) test case so we can report Reason properly 1245 [{?MODULE,error_in_suite,[[{error,Reason}]]}]; 1246 {error,{bad_return,_Bad}} -> 1247 Reason = 1248 list_to_atom("Bad return value from "++ 1249 atom_to_list(Mod)++":all/0"), 1250 [{?MODULE,error_in_suite,[[{error,Reason}]]}]; 1251 {error,{bad_hook_return,Bad}} -> 1252 Reason = 1253 list_to_atom("Bad return value from post_all/3 hook function"), 1254 [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}]; 1255 {error,{failed,ExitReason}} -> 1256 case ct_util:get_testdata({error_in_suite,Mod}) of 1257 undefined -> 1258 ErrStr = io_lib:format("~n*** ERROR *** " 1259 "~w:all/0 failed: ~tp~n", 1260 [Mod,ExitReason]), 1261 io:format(?def_gl, ErrStr, []), 1262 %% save the error info so it doesn't get printed twice 1263 ct_util:set_testdata_async({{error_in_suite,Mod}, 1264 ExitReason}); 1265 _ExitReason -> 1266 ct_util:delete_testdata({error_in_suite,Mod}) 1267 end, 1268 Reason = list_to_atom(atom_to_list(Mod)++":all/0 failed"), 1269 %% this makes test_server call error_in_suite as first 1270 %% (and only) test case so we can report Reason properly 1271 [{?MODULE,error_in_suite,[[{error,Reason}]]}]; 1272 {error,What} -> 1273 [{?MODULE,error_in_suite,[[{error,What}]]}] 1274 end. 1275 1276%%!============================================================ 1277%%! The support for sequences by means of using sequences/0 1278%%! will be removed in OTP R15. The code below is only kept 1279%%! for backwards compatibility. From OTP R13 groups with 1280%%! sequence property should be used instead! 1281%%!============================================================ 1282%%!============================================================ 1283%%! START OF DEPRECATED SUPPORT FOR SEQUENCES ---> 1284 1285get_seq(Mod, Func) -> 1286 case ct_util:read_suite_data({seq,Mod,Func}) of 1287 undefined -> 1288 case catch apply(Mod,sequences,[]) of 1289 {'EXIT',_} -> 1290 []; 1291 Seqs -> 1292 case lists:keysearch(Func,1,Seqs) of 1293 {value,{Func,SeqTCs}} -> 1294 case catch save_seq(Mod,Func,SeqTCs) of 1295 {error,What} -> 1296 [{?MODULE,error_in_suite,[[{error,What}]]}]; 1297 _ -> 1298 SeqTCs 1299 end; 1300 false -> 1301 [] 1302 end 1303 end; 1304 TCs when is_list(TCs) -> 1305 TCs; 1306 _ -> 1307 [] 1308 end. 1309 1310save_seqs(Mod,AllTCs) -> 1311 case lists:keymember(sequence,1,AllTCs) of 1312 true -> 1313 case catch apply(Mod,sequences,[]) of 1314 {'EXIT',_} -> 1315 Reason = list_to_atom(atom_to_list(Mod)++ 1316 ":sequences/0 is missing"), 1317 throw({error,Reason}); 1318 Seqs -> 1319 save_seqs(Mod,AllTCs,Seqs,AllTCs) 1320 end; 1321 false -> 1322 AllTCs 1323 end. 1324 1325save_seqs(Mod,[{sequence,Seq}|TCs],Seqs,All) -> 1326 case lists:keysearch(Seq,1,Seqs) of 1327 {value,{Seq,SeqTCs}} -> 1328 save_seq(Mod,Seq,SeqTCs,All), 1329 [Seq|save_seqs(Mod,TCs,Seqs,All)]; 1330 false -> 1331 Reason = list_to_atom( 1332 atom_to_list(Seq)++" is missing in "++ 1333 atom_to_list(Mod)), 1334 throw({error,Reason}) 1335 end; 1336save_seqs(Mod,[TC|TCs],Seqs,All) -> 1337 [TC|save_seqs(Mod,TCs,Seqs,All)]; 1338save_seqs(_,[],_,_) -> 1339 []. 1340 1341save_seq(Mod,Seq,SeqTCs) -> 1342 save_seq(Mod,Seq,SeqTCs,apply(Mod,all,[])). 1343 1344save_seq(Mod,Seq,SeqTCs,All) -> 1345 check_private(Seq,SeqTCs,All), 1346 check_multiple(Mod,Seq,SeqTCs), 1347 ct_util:save_suite_data({seq,Mod,Seq},SeqTCs), 1348 lists:foreach(fun(TC) -> 1349 ct_util:save_suite_data({seq,Mod,TC},Seq) 1350 end, SeqTCs). 1351 1352check_private(Seq,TCs,All) -> 1353 Bad = lists:filter(fun(TC) -> lists:member(TC,All) end, TCs), 1354 if Bad /= [] -> 1355 Reason = io_lib:format("regular test cases not allowed in sequence ~tp: " 1356 "~tp",[Seq,Bad]), 1357 throw({error,list_to_atom(lists:flatten(Reason))}); 1358 true -> 1359 ok 1360 end. 1361 1362check_multiple(Mod,Seq,TCs) -> 1363 Bad = lists:filter(fun(TC) -> 1364 case ct_util:read_suite_data({seq,Mod,TC}) of 1365 Seq1 when Seq1 /= undefined, Seq1 /= Seq -> 1366 true; 1367 1368 _ -> false 1369 end 1370 end,TCs), 1371 if Bad /= [] -> 1372 Reason = io_lib:format("test cases found in multiple sequences: " 1373 "~tp",[Bad]), 1374 throw({error,list_to_atom(lists:flatten(Reason))}); 1375 true -> 1376 ok 1377 end. 1378 1379%%! <--- END OF DEPRECATED SUPPORT FOR SEQUENCES 1380%%!============================================================ 1381 1382%% let test_server call this function as a testcase only so that 1383%% the user may see info about what's missing in the suite 1384error_in_suite(Config) -> 1385 Reason = test_server:lookup_config(error,Config), 1386 exit(Reason). 1387 1388%% if init_per_suite and end_per_suite are missing in the suite, 1389%% these will be called instead (without any trace of them in the 1390%% log files), only so that it's possible to call hook functions 1391%% for configuration 1392init_per_suite(Config) -> 1393 Config. 1394 1395end_per_suite(_Config) -> 1396 ok. 1397 1398%% if the group config functions are missing in the suite, 1399%% use these instead 1400init_per_group(GroupName, Config) -> 1401 ct:comment(io_lib:format("start of ~tp", [GroupName])), 1402 ct_logs:log("TEST INFO", "init_per_group/2 for ~tw missing " 1403 "in suite, using default.", 1404 [GroupName]), 1405 Config. 1406 1407end_per_group(GroupName, _) -> 1408 ct:comment(io_lib:format("end of ~tp", [GroupName])), 1409 ct_logs:log("TEST INFO", "end_per_group/2 for ~tw missing " 1410 "in suite, using default.", 1411 [GroupName]), 1412 ok. 1413 1414%%%----------------------------------------------------------------- 1415%%% -spec report(What,Data) -> ok 1416report(What,Data) -> 1417 case What of 1418 loginfo -> 1419 %% logfiles and direcories have been created for a test and the 1420 %% top level test index page needs to be refreshed 1421 TestName = filename:basename(?val(topdir, Data), ".logs"), 1422 RunDir = ?val(rundir, Data), 1423 _ = ct_logs:make_all_suites_index({TestName,RunDir}), 1424 ok; 1425 tests_start -> 1426 ok; 1427 tests_done -> 1428 ok; 1429 severe_error -> 1430 ct_event:sync_notify(#event{name=What, 1431 node=node(), 1432 data=Data}), 1433 ct_util:set_testdata({What,Data}), 1434 ok; 1435 tc_start -> 1436 %% Data = {{Suite,{Func,GroupName}},LogFileName} 1437 Data1 = case Data of 1438 {{Suite,{Func,undefined}},LFN} -> {{Suite,Func},LFN}; 1439 _ -> Data 1440 end, 1441 ct_event:sync_notify(#event{name=tc_logfile, 1442 node=node(), 1443 data=Data1}), 1444 ok; 1445 tc_done -> 1446 {Suite,{Func,GrName},Result} = Data, 1447 FuncSpec = if GrName == undefined -> Func; 1448 true -> {Func,GrName} 1449 end, 1450 %% Register the group leader for the process calling the report 1451 %% function, making it possible for a hook function to print 1452 %% in the test case log file 1453 ReportingPid = self(), 1454 ct_logs:register_groupleader(ReportingPid, group_leader()), 1455 case Result of 1456 {failed, Reason} -> 1457 ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason}); 1458 {skipped,{failed,{_,init_per_testcase,_}}=Reason} -> 1459 ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); 1460 {skipped,{require_failed,_}=Reason} -> 1461 ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); 1462 {skipped,Reason} -> 1463 ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason}); 1464 {auto_skipped,Reason} -> 1465 ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); 1466 _Else -> 1467 ok 1468 end, 1469 ct_logs:unregister_groupleader(ReportingPid), 1470 case {Func,Result} of 1471 {error_in_suite,_} when Suite == ?MODULE -> 1472 ok; 1473 {init_per_suite,_} -> 1474 ok; 1475 {end_per_suite,_} -> 1476 ok; 1477 {init_per_group,_} -> 1478 ok; 1479 {end_per_group,_} -> 1480 ok; 1481 {_,ok} -> 1482 add_to_stats(ok); 1483 {_,{skipped,{failed,{_,init_per_testcase,_}}}} -> 1484 add_to_stats(auto_skipped); 1485 {_,{skipped,{require_failed,_}}} -> 1486 add_to_stats(auto_skipped); 1487 {_,{skipped,{timetrap_error,_}}} -> 1488 add_to_stats(auto_skipped); 1489 {_,{skipped,{invalid_time_format,_}}} -> 1490 add_to_stats(auto_skipped); 1491 {_,{skipped,_}} -> 1492 add_to_stats(user_skipped); 1493 {_,{auto_skipped,_}} -> 1494 add_to_stats(auto_skipped); 1495 {_,{SkipOrFail,_Reason}} -> 1496 add_to_stats(SkipOrFail) 1497 end; 1498 tc_user_skip -> 1499 %% test case or config function specified as skipped in testspec, 1500 %% or init config func for suite/group has returned {skip,Reason} 1501 %% Data = {Suite,{Func,GroupName},Comment} 1502 {Func,Data1} = case Data of 1503 {Suite,{F,undefined},Comment} -> 1504 {F,{Suite,F,Comment}}; 1505 D = {_,{F,_},_} -> 1506 {F,D} 1507 end, 1508 ct_event:sync_notify(#event{name=tc_user_skip, 1509 node=node(), 1510 data=Data1}), 1511 ct_hooks:on_tc_skip(What, Data1), 1512 if Func /= init_per_suite, Func /= init_per_group, 1513 Func /= end_per_suite, Func /= end_per_group -> 1514 add_to_stats(user_skipped); 1515 true -> 1516 ok 1517 end; 1518 tc_auto_skip -> 1519 %% test case skipped because of error in config function, or 1520 %% config function skipped because of error in info function 1521 %% Data = {Suite,{Func,GroupName},Comment} 1522 {Func,Data1} = case Data of 1523 {Suite,{F,undefined},Comment} -> 1524 {F,{Suite,F,Comment}}; 1525 D = {_,{F,_},_} -> 1526 {F,D} 1527 end, 1528 %% this test case does not have a log, so printouts 1529 %% from event handlers should end up in the main log 1530 ct_event:sync_notify(#event{name=tc_auto_skip, 1531 node=node(), 1532 data=Data1}), 1533 ct_hooks:on_tc_skip(What, Data1), 1534 if Func /= end_per_suite, 1535 Func /= end_per_group -> 1536 add_to_stats(auto_skipped); 1537 true -> 1538 ok 1539 end; 1540 framework_error -> 1541 case Data of 1542 {{M,F},E} -> 1543 ct_event:sync_notify(#event{name=tc_done, 1544 node=node(), 1545 data={M,F,{framework_error,E}}}); 1546 _ -> 1547 ct_event:sync_notify(#event{name=tc_done, 1548 node=node(), 1549 data=Data}) 1550 end; 1551 _ -> 1552 ok 1553 end, 1554 catch vts:report(What,Data). 1555 1556add_to_stats(Result) -> 1557 Update = fun({Ok,Failed,Skipped={UserSkipped,AutoSkipped}}) -> 1558 Stats = 1559 case Result of 1560 ok -> 1561 {Ok+1,Failed,Skipped}; 1562 failed -> 1563 {Ok,Failed+1,Skipped}; 1564 skipped -> 1565 {Ok,Failed,{UserSkipped+1,AutoSkipped}}; 1566 user_skipped -> 1567 {Ok,Failed,{UserSkipped+1,AutoSkipped}}; 1568 auto_skipped -> 1569 {Ok,Failed,{UserSkipped,AutoSkipped+1}} 1570 end, 1571 ct_event:sync_notify(#event{name=test_stats, 1572 node=node(), 1573 data=Stats}), 1574 Stats 1575 end, 1576 ct_util:update_testdata(stats, Update). 1577 1578%%%----------------------------------------------------------------- 1579%%% -spec warn(What) -> true | false 1580warn(What) when What==nodes; What==processes -> 1581 false; 1582warn(_What) -> 1583 true. 1584 1585%%%----------------------------------------------------------------- 1586%%% -spec add_data_dir(File0, Config) -> File1 1587add_data_dir(File,Config) when is_atom(File) -> 1588 add_data_dir(atom_to_list(File),Config); 1589 1590add_data_dir(File,Config) when is_list(File) -> 1591 case filename:split(File) of 1592 [File] -> 1593 %% no user path, add data dir 1594 case lists:keysearch(data_dir,1,Config) of 1595 {value,{data_dir,DataDir}} -> 1596 filename:join(DataDir,File); 1597 _ -> 1598 File 1599 end; 1600 _ -> 1601 File 1602 end. 1603 1604%%%----------------------------------------------------------------- 1605%%% -spec get_logopts() -> [LogOpt] 1606get_logopts() -> 1607 case ct_util:get_testdata(logopts) of 1608 undefined -> 1609 []; 1610 LogOpts -> 1611 LogOpts 1612 end. 1613 1614%%%----------------------------------------------------------------- 1615%%% -spec format_comment(Comment) -> HtmlComment 1616format_comment(Comment) -> 1617 "<font color=\"green\">" ++ Comment ++ "</font>". 1618 1619%%%----------------------------------------------------------------- 1620%%% -spec get_html_wrapper(TestName, PrintLabel, Cwd) -> Header 1621get_html_wrapper(TestName, PrintLabel, Cwd, TableCols) -> 1622 get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, utf8). 1623 1624get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding) -> 1625 ct_logs:get_ts_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding). 1626 1627%%%----------------------------------------------------------------- 1628%%% -spec get_log_dir() -> {ok,LogDir} 1629get_log_dir() -> 1630 ct_logs:get_log_dir(true). 1631 1632%%%----------------------------------------------------------------- 1633%%% Call all and group callbacks and post_* hooks with error handling 1634safe_apply_all_0(Mod) -> 1635 try apply(Mod, all, []) of 1636 AllTCs0 when is_list(AllTCs0) -> 1637 try save_seqs(Mod,AllTCs0) of 1638 SeqsAndTCs when is_list(SeqsAndTCs) -> 1639 all_hook(Mod,SeqsAndTCs) 1640 catch throw:{error,What} -> 1641 {error,What} 1642 end; 1643 {skip,_}=Skip -> 1644 all_hook(Mod,Skip); 1645 Bad -> 1646 {error,{bad_return,Bad}} 1647 catch 1648 _:Reason:Stacktrace -> 1649 handle_callback_crash(Reason,Stacktrace,Mod,all,{error,undef}) 1650 end. 1651 1652all_hook(Mod, All) -> 1653 case ct_hooks:all(Mod, All) of 1654 AllTCs when is_list(AllTCs) -> 1655 {ok,AllTCs}; 1656 {skip,_}=Skip -> 1657 Skip; 1658 {fail,Reason} -> 1659 {error,Reason}; 1660 Bad -> 1661 {error,{bad_hook_return,Bad}} 1662 end. 1663 1664safe_apply_groups_0(Mod,Default) -> 1665 try apply(Mod, groups, []) of 1666 GroupDefs when is_list(GroupDefs) -> 1667 case ct_hooks:groups(Mod, GroupDefs) of 1668 GroupDefs1 when is_list(GroupDefs1) -> 1669 {ok,GroupDefs1}; 1670 {fail,Reason} -> 1671 {error,Reason}; 1672 Bad -> 1673 {error,{bad_hook_return,Bad}} 1674 end; 1675 Bad -> 1676 {error,{bad_return,Bad}} 1677 catch 1678 _:Reason:Stacktrace -> 1679 handle_callback_crash(Reason,Stacktrace,Mod,groups,Default) 1680 end. 1681 1682handle_callback_crash(undef,[{Mod,Func,[],_}|_],Mod,Func,Default) -> 1683 case ct_hooks:Func(Mod, []) of 1684 [] -> 1685 Default; 1686 List when is_list(List) -> 1687 {ok,List}; 1688 {fail,Reason} -> 1689 {error,Reason}; 1690 Bad -> 1691 {error,{bad_hook_return,Bad}} 1692 end; 1693handle_callback_crash(Reason,Stacktrace,_Mod,_Func,_Default) -> 1694 {error,{failed,{Reason,Stacktrace}}}. 1695 1696expand_tests(Mod, [{testcase,Case,[Prop]}|Tests]) -> 1697 [{repeat,{Mod,Case},Prop}|expand_tests(Mod,Tests)]; 1698expand_tests(Mod,[Test|Tests]) -> 1699 [Test|expand_tests(Mod,Tests)]; 1700expand_tests(_Mod,[]) -> 1701 []. 1702