1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2004-2020. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20 21-module(ct_run). 22 23%% Script interface 24-export([script_start/0,script_usage/0]). 25 26%% User interface 27-export([install/1,install/2,run/1,run/2,run/3,run_test/1, 28 run_testspec/1,step/3,step/4,refresh_logs/1]). 29 30%% Misc internal API functions 31-export([variables_file_name/1,script_start1/2,run_test2/1, run_make/3]). 32 33-include("ct.hrl"). 34-include("ct_event.hrl"). 35-include("ct_util.hrl"). 36 37-define(abs(Name), filename:absname(Name)). 38-define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)). 39 40-define(EXIT_STATUS_TEST_SUCCESSFUL, 0). 41-define(EXIT_STATUS_TEST_CASE_FAILED, 1). 42-define(EXIT_STATUS_TEST_RUN_FAILED, 2). 43 44-define(default_verbosity, [{default,?MAX_VERBOSITY}, 45 {'$unspecified',?MAX_VERBOSITY}]). 46 47-record(opts, {label, 48 profile, 49 shell, 50 cover, 51 cover_stop, 52 coverspec, 53 step, 54 logdir, 55 logopts = [], 56 basic_html, 57 esc_chars = true, 58 verbosity = [], 59 config = [], 60 event_handlers = [], 61 ct_hooks = [], 62 enable_builtin_hooks, 63 include = [], 64 auto_compile, 65 abort_if_missing_suites, 66 silent_connections = [], 67 stylesheet, 68 multiply_timetraps, 69 scale_timetraps, 70 create_priv_dir, 71 testspec_files = [], 72 current_testspec, 73 tests, 74 starter}). 75 76script_start() -> 77 process_flag(trap_exit, true), 78 Init = init:get_arguments(), 79 CtArgs = lists:takewhile(fun({ct_erl_args,_}) -> false; 80 (_) -> true end, Init), 81 82 %% convert relative dirs added with pa or pz (pre erl_args on 83 %% the ct_run command line) to absolute so that app modules 84 %% can be found even after CT changes CWD to logdir 85 rel_to_abs(CtArgs), 86 87 Args = 88 case application:get_env(common_test, run_test_start_opts) of 89 {ok,EnvStartOpts} -> 90 FlagFilter = fun(Flags) -> 91 lists:filter(fun({root,_}) -> false; 92 ({progname,_}) -> false; 93 ({home,_}) -> false; 94 ({noshell,_}) -> false; 95 ({noinput,_}) -> false; 96 (_) -> true 97 end, Flags) 98 end, 99 %% used for purpose of testing the run_test interface 100 io:format(user, "~n-------------------- START ARGS " 101 "--------------------~n", []), 102 io:format(user, "--- Init args:~n~tp~n", [FlagFilter(Init)]), 103 io:format(user, "--- CT args:~n~tp~n", [FlagFilter(CtArgs)]), 104 EnvArgs = opts2args(EnvStartOpts), 105 io:format(user, "--- Env opts -> args:~n~tp~n =>~n~tp~n", 106 [EnvStartOpts,EnvArgs]), 107 Merged = merge_arguments(CtArgs ++ EnvArgs), 108 io:format(user, "--- Merged args:~n~tp~n", [FlagFilter(Merged)]), 109 io:format(user, "-----------------------------------" 110 "-----------------~n~n", []), 111 Merged; 112 _ -> 113 merge_arguments(CtArgs) 114 end, 115 case proplists:get_value(help, Args) of 116 undefined -> script_start(Args); 117 _ -> script_usage() 118 end. 119 120script_start(Args) -> 121 Tracing = start_trace(Args), 122 case ct_repeat:loop_test(script, Args) of 123 false -> 124 {ok,Cwd} = file:get_cwd(), 125 CTVsn = 126 case filename:basename(code:lib_dir(common_test)) of 127 CTBase when is_list(CTBase) -> 128 case string:lexemes(CTBase, "-") of 129 ["common_test",Vsn] -> " v"++Vsn; 130 _ -> "" 131 end 132 end, 133 io:format("~nCommon Test~s starting (cwd is ~ts)~n~n", 134 [CTVsn,Cwd]), 135 Self = self(), 136 Pid = spawn_link(fun() -> script_start1(Self, Args) end), 137 receive 138 {'EXIT',Pid,Reason} -> 139 case Reason of 140 {user_error,What} -> 141 io:format("\nTest run failed!\nReason: ~tp\n\n\n", 142 [What]), 143 finish(Tracing, ?EXIT_STATUS_TEST_RUN_FAILED, Args); 144 _ -> 145 io:format("Test run crashed! " 146 "This could be an internal error " 147 "- please report!\n\n" 148 "~tp\n\n\n", [Reason]), 149 finish(Tracing, ?EXIT_STATUS_TEST_RUN_FAILED, Args) 150 end; 151 {Pid,{error,Reason}} -> 152 io:format("\nTest run failed! Reason:\n~tp\n\n\n",[Reason]), 153 finish(Tracing, ?EXIT_STATUS_TEST_RUN_FAILED, Args); 154 {Pid,Result} -> 155 io:nl(), 156 finish(Tracing, analyze_test_result(Result, Args), Args) 157 end; 158 {error,_LoopReason} -> 159 finish(Tracing, ?EXIT_STATUS_TEST_RUN_FAILED, Args); 160 Result -> 161 io:nl(), 162 finish(Tracing, analyze_test_result(Result, Args), Args) 163 end. 164 165%% analyze the result of one test run, or many (in case of looped test) 166analyze_test_result(ok, _) -> 167 ?EXIT_STATUS_TEST_SUCCESSFUL; 168analyze_test_result({error,_Reason}, _) -> 169 ?EXIT_STATUS_TEST_RUN_FAILED; 170analyze_test_result({_Ok,Failed,{_UserSkipped,AutoSkipped}}, Args) -> 171 if Failed > 0 -> 172 ?EXIT_STATUS_TEST_CASE_FAILED; 173 true -> 174 case AutoSkipped of 175 0 -> 176 ?EXIT_STATUS_TEST_SUCCESSFUL; 177 _ -> 178 case get_start_opt(exit_status, 179 fun([ExitOpt]) -> ExitOpt end, 180 Args) of 181 undefined -> 182 ?EXIT_STATUS_TEST_CASE_FAILED; 183 "ignore_config" -> 184 ?EXIT_STATUS_TEST_SUCCESSFUL 185 end 186 end 187 end; 188analyze_test_result([Result|Rs], Args) -> 189 case analyze_test_result(Result, Args) of 190 ?EXIT_STATUS_TEST_SUCCESSFUL -> 191 analyze_test_result(Rs, Args); 192 Other -> 193 Other 194 end; 195analyze_test_result([], _) -> 196 ?EXIT_STATUS_TEST_SUCCESSFUL; 197analyze_test_result(interactive_mode, _) -> 198 interactive_mode; 199analyze_test_result(Unknown, _) -> 200 io:format("\nTest run failed! Reason:\n~tp\n\n\n",[Unknown]), 201 ?EXIT_STATUS_TEST_RUN_FAILED. 202 203finish(Tracing, ExitStatus, Args) -> 204 stop_trace(Tracing), 205 timer:sleep(1000), 206 if ExitStatus == interactive_mode -> 207 interactive_mode; 208 true -> 209 %% it's possible to tell CT to finish execution with a call 210 %% to a different function than the normal halt/1 BIF 211 %% (meant to be used mainly for reading the CT exit status) 212 case get_start_opt(halt_with, 213 fun([HaltMod,HaltFunc]) -> 214 {list_to_atom(HaltMod), 215 list_to_atom(HaltFunc)} end, 216 Args) of 217 undefined -> 218 halt(ExitStatus); 219 {M,F} -> 220 apply(M, F, [ExitStatus]) 221 end 222 end. 223 224script_start1(Parent, Args) -> 225 %% tag this process 226 ct_util:mark_process(), 227 %% read general start flags 228 Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args), 229 Profile = get_start_opt(profile, fun([Prof]) -> Prof end, Args), 230 Shell = get_start_opt(shell, true, Args), 231 Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), 232 CoverStop = get_start_opt(cover_stop, 233 fun([CS]) -> list_to_atom(CS) end, Args), 234 LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), 235 LogOpts = get_start_opt(logopts, 236 fun(Os) -> [list_to_atom(O) || O <- Os] end, 237 [], Args), 238 Verbosity = verbosity_args2opts(Args), 239 MultTT = get_start_opt(multiply_timetraps, 240 fun([MT]) -> list_to_integer(MT) end, Args), 241 ScaleTT = get_start_opt(scale_timetraps, 242 fun([CT]) -> list_to_atom(CT); 243 ([]) -> true 244 end, Args), 245 CreatePrivDir = get_start_opt(create_priv_dir, 246 fun([PD]) -> list_to_atom(PD); 247 ([]) -> auto_per_tc 248 end, Args), 249 EvHandlers = event_handler_args2opts(Args), 250 CTHooks = ct_hooks_args2opts(Args), 251 EnableBuiltinHooks = get_start_opt(enable_builtin_hooks, 252 fun([CT]) -> list_to_atom(CT); 253 ([]) -> undefined 254 end, undefined, Args), 255 256 %% check flags and set corresponding application env variables 257 258 %% ct_decrypt_key | ct_decrypt_file 259 case proplists:get_value(ct_decrypt_key, Args) of 260 [DecryptKey] -> 261 application:set_env(common_test, decrypt, {key,DecryptKey}); 262 undefined -> 263 case proplists:get_value(ct_decrypt_file, Args) of 264 [DecryptFile] -> 265 application:set_env(common_test, decrypt, 266 {file,?abs(DecryptFile)}); 267 undefined -> 268 application:unset_env(common_test, decrypt) 269 end 270 end, 271 %% no_auto_compile + include 272 {AutoCompile,IncludeDirs} = 273 case proplists:get_value(no_auto_compile, Args) of 274 undefined -> 275 application:set_env(common_test, auto_compile, true), 276 InclDirs = 277 case proplists:get_value(include, Args) of 278 Incls when is_list(hd(Incls)) -> 279 [filename:absname(IDir) || IDir <- Incls]; 280 Incl when is_list(Incl) -> 281 [filename:absname(Incl)]; 282 undefined -> 283 [] 284 end, 285 case os:getenv("CT_INCLUDE_PATH") of 286 false -> 287 application:set_env(common_test, include, InclDirs), 288 {undefined,InclDirs}; 289 CtInclPath -> 290 AllInclDirs = 291 string:lexemes(CtInclPath,[$:,$ ,$,]) ++ InclDirs, 292 application:set_env(common_test, include, AllInclDirs), 293 {undefined,AllInclDirs} 294 end; 295 _ -> 296 application:set_env(common_test, auto_compile, false), 297 {false,[]} 298 end, 299 300 %% abort test run if some suites can't be compiled 301 AbortIfMissing = get_start_opt(abort_if_missing_suites, 302 fun([]) -> true; 303 ([Bool]) -> list_to_atom(Bool) 304 end, false, Args), 305 %% silent connections 306 SilentConns = 307 get_start_opt(silent_connections, 308 fun(["all"]) -> [all]; 309 (Conns) -> [list_to_atom(Conn) || Conn <- Conns] 310 end, [], Args), 311 %% stylesheet 312 Stylesheet = get_start_opt(stylesheet, 313 fun([SS]) -> ?abs(SS) end, Args), 314 %% basic_html - used by ct_logs 315 BasicHtml = case proplists:get_value(basic_html, Args) of 316 undefined -> 317 application:set_env(common_test, basic_html, false), 318 undefined; 319 _ -> 320 application:set_env(common_test, basic_html, true), 321 true 322 end, 323 %% esc_chars - used by ct_logs 324 EscChars = case proplists:get_value(no_esc_chars, Args) of 325 undefined -> 326 application:set_env(common_test, esc_chars, true), 327 undefined; 328 _ -> 329 application:set_env(common_test, esc_chars, false), 330 false 331 end, 332 %% disable_log_cache - used by ct_logs 333 case proplists:get_value(disable_log_cache, Args) of 334 undefined -> 335 application:set_env(common_test, disable_log_cache, false); 336 _ -> 337 application:set_env(common_test, disable_log_cache, true) 338 end, 339 %% log_cleanup - used by ct_logs 340 KeepLogs = get_start_opt(keep_logs, 341 fun ct_logs:parse_keep_logs/1, 342 all, 343 Args), 344 application:set_env(common_test, keep_logs, KeepLogs), 345 346 Opts = #opts{label = Label, profile = Profile, 347 shell = Shell, 348 cover = Cover, cover_stop = CoverStop, 349 logdir = LogDir, logopts = LogOpts, 350 basic_html = BasicHtml, 351 esc_chars = EscChars, 352 verbosity = Verbosity, 353 event_handlers = EvHandlers, 354 ct_hooks = CTHooks, 355 enable_builtin_hooks = EnableBuiltinHooks, 356 auto_compile = AutoCompile, 357 abort_if_missing_suites = AbortIfMissing, 358 include = IncludeDirs, 359 silent_connections = SilentConns, 360 stylesheet = Stylesheet, 361 multiply_timetraps = MultTT, 362 scale_timetraps = ScaleTT, 363 create_priv_dir = CreatePrivDir, 364 starter = script}, 365 366 %% check if log files should be refreshed or go on to run tests... 367 Result = run_or_refresh(Opts, Args), 368 369 %% send final results to starting process waiting in script_start/0 370 Parent ! {self(), Result}. 371 372run_or_refresh(Opts = #opts{logdir = LogDir}, Args) -> 373 case proplists:get_value(refresh_logs, Args) of 374 undefined -> 375 script_start2(Opts, Args); 376 Refresh -> 377 LogDir1 = case Refresh of 378 [] -> which(logdir,LogDir); 379 [RefreshDir] -> ?abs(RefreshDir) 380 end, 381 {ok,Cwd} = file:get_cwd(), 382 ok = file:set_cwd(LogDir1), 383 %% give the shell time to print version etc 384 timer:sleep(500), 385 io:nl(), 386 case catch ct_logs:make_all_runs_index(refresh) of 387 {'EXIT',ARReason} -> 388 ok = file:set_cwd(Cwd), 389 {error,{all_runs_index,ARReason}}; 390 _ -> 391 case catch ct_logs:make_all_suites_index(refresh) of 392 {'EXIT',ASReason} -> 393 ok = file:set_cwd(Cwd), 394 {error,{all_suites_index,ASReason}}; 395 _ -> 396 ok = file:set_cwd(Cwd), 397 io:format("Logs in ~ts refreshed!~n~n", 398 [LogDir1]), 399 timer:sleep(500), % time to flush io before quitting 400 ok 401 end 402 end 403 end. 404 405script_start2(Opts = #opts{shell = undefined}, Args) -> 406 case proplists:get_value(spec, Args) of 407 Specs when Specs =/= [], Specs =/= undefined -> 408 Specs1 = get_start_opt(join_specs, [Specs], Specs, Args), 409 %% using testspec as input for test 410 Relaxed = get_start_opt(allow_user_terms, true, false, Args), 411 try ct_testspec:collect_tests_from_file(Specs1, Relaxed) of 412 TestSpecData -> 413 execute_all_specs(TestSpecData, Opts, Args, []) 414 catch 415 throw:{error,Reason}:StackTrace -> 416 {error,{invalid_testspec,{Reason,StackTrace}}}; 417 _:Reason:StackTrace -> 418 {error,{invalid_testspec,{Reason,StackTrace}}} 419 end; 420 [] -> 421 {error,no_testspec_specified}; 422 _ -> % no testspec used 423 %% read config/userconfig from start flags 424 InitConfig = ct_config:prepare_config_list(Args), 425 TheLogDir = which(logdir, Opts#opts.logdir), 426 case check_and_install_configfiles(InitConfig, 427 TheLogDir, 428 Opts) of 429 ok -> % go on read tests from start flags 430 script_start3(Opts#opts{config=InitConfig, 431 logdir=TheLogDir}, Args); 432 Error -> 433 Error 434 end 435 end; 436 437script_start2(Opts, Args) -> 438 %% read config/userconfig from start flags 439 InitConfig = ct_config:prepare_config_list(Args), 440 LogDir = which(logdir, Opts#opts.logdir), 441 case check_and_install_configfiles(InitConfig, LogDir, Opts) of 442 ok -> % go on read tests from start flags 443 script_start3(Opts#opts{config=InitConfig, 444 logdir=LogDir}, Args); 445 Error -> 446 Error 447 end. 448 449execute_all_specs([], _, _, Result) -> 450 Result1 = lists:reverse(Result), 451 case lists:keysearch('EXIT', 1, Result1) of 452 {value,{_,_,ExitReason}} -> 453 exit(ExitReason); 454 false -> 455 case lists:keysearch(error, 1, Result1) of 456 {value,Error} -> 457 Error; 458 false -> 459 lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, 460 {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> 461 {Ok1+Ok,Fail1+Fail, 462 {UserSkip1+UserSkip, 463 AutoSkip1+AutoSkip}} 464 end, {0,0,{0,0}}, Result1) 465 end 466 end; 467 468execute_all_specs([{Specs,TS} | TSs], Opts, Args, Result) -> 469 CombinedOpts = combine_test_opts(TS, Specs, Opts), 470 try execute_one_spec(TS, CombinedOpts, Args) of 471 ExecResult -> 472 execute_all_specs(TSs, Opts, Args, [ExecResult|Result]) 473 catch 474 _ : ExitReason -> 475 execute_all_specs(TSs, Opts, Args, 476 [{'EXIT',self(),ExitReason}|Result]) 477 end. 478 479execute_one_spec(TS, Opts, Args) -> 480 %% read config/userconfig from start flags 481 InitConfig = ct_config:prepare_config_list(Args), 482 TheLogDir = which(logdir, Opts#opts.logdir), 483 %% merge config from start flags with config from testspec 484 AllConfig = merge_vals([InitConfig, Opts#opts.config]), 485 case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of 486 ok -> % read tests from spec 487 {Run,Skip} = ct_testspec:prepare_tests(TS, node()), 488 Result = do_run(Run, Skip, Opts#opts{config=AllConfig, 489 logdir=TheLogDir, 490 current_testspec=TS}, Args), 491 ct_util:delete_testdata(testspec), 492 Result; 493 Error -> 494 Error 495 end. 496 497combine_test_opts(TS, Specs, Opts) -> 498 TSOpts = get_data_for_node(TS, node()), 499 500 Label = choose_val(Opts#opts.label, 501 TSOpts#opts.label), 502 503 Profile = choose_val(Opts#opts.profile, 504 TSOpts#opts.profile), 505 506 LogDir = choose_val(Opts#opts.logdir, 507 TSOpts#opts.logdir), 508 509 AllLogOpts = merge_vals([Opts#opts.logopts, 510 TSOpts#opts.logopts]), 511 AllVerbosity = 512 merge_keyvals([Opts#opts.verbosity, 513 TSOpts#opts.verbosity]), 514 AllSilentConns = 515 merge_vals([Opts#opts.silent_connections, 516 TSOpts#opts.silent_connections]), 517 Cover = 518 choose_val(Opts#opts.cover, 519 TSOpts#opts.cover), 520 CoverStop = 521 choose_val(Opts#opts.cover_stop, 522 TSOpts#opts.cover_stop), 523 MultTT = 524 choose_val(Opts#opts.multiply_timetraps, 525 TSOpts#opts.multiply_timetraps), 526 ScaleTT = 527 choose_val(Opts#opts.scale_timetraps, 528 TSOpts#opts.scale_timetraps), 529 530 CreatePrivDir = 531 choose_val(Opts#opts.create_priv_dir, 532 TSOpts#opts.create_priv_dir), 533 534 AllEvHs = 535 merge_vals([Opts#opts.event_handlers, 536 TSOpts#opts.event_handlers]), 537 538 AllCTHooks = merge_vals( 539 [Opts#opts.ct_hooks, 540 TSOpts#opts.ct_hooks]), 541 542 EnableBuiltinHooks = 543 choose_val( 544 Opts#opts.enable_builtin_hooks, 545 TSOpts#opts.enable_builtin_hooks), 546 547 Stylesheet = 548 choose_val(Opts#opts.stylesheet, 549 TSOpts#opts.stylesheet), 550 551 AllInclude = merge_vals([Opts#opts.include, 552 TSOpts#opts.include]), 553 application:set_env(common_test, include, AllInclude), 554 555 AutoCompile = 556 case choose_val(Opts#opts.auto_compile, 557 TSOpts#opts.auto_compile) of 558 undefined -> 559 true; 560 ACBool -> 561 application:set_env(common_test, 562 auto_compile, 563 ACBool), 564 ACBool 565 end, 566 567 AbortIfMissing = choose_val(Opts#opts.abort_if_missing_suites, 568 TSOpts#opts.abort_if_missing_suites), 569 570 BasicHtml = 571 case choose_val(Opts#opts.basic_html, 572 TSOpts#opts.basic_html) of 573 undefined -> 574 false; 575 BHBool -> 576 application:set_env(common_test, basic_html, 577 BHBool), 578 BHBool 579 end, 580 581 EscChars = 582 case choose_val(Opts#opts.esc_chars, 583 TSOpts#opts.esc_chars) of 584 undefined -> 585 true; 586 ECBool -> 587 application:set_env(common_test, esc_chars, 588 ECBool), 589 ECBool 590 end, 591 592 Opts#opts{label = Label, 593 profile = Profile, 594 testspec_files = Specs, 595 cover = Cover, 596 cover_stop = CoverStop, 597 logdir = which(logdir, LogDir), 598 logopts = AllLogOpts, 599 basic_html = BasicHtml, 600 esc_chars = EscChars, 601 verbosity = AllVerbosity, 602 silent_connections = AllSilentConns, 603 config = TSOpts#opts.config, 604 event_handlers = AllEvHs, 605 ct_hooks = AllCTHooks, 606 enable_builtin_hooks = EnableBuiltinHooks, 607 stylesheet = Stylesheet, 608 auto_compile = AutoCompile, 609 abort_if_missing_suites = AbortIfMissing, 610 include = AllInclude, 611 multiply_timetraps = MultTT, 612 scale_timetraps = ScaleTT, 613 create_priv_dir = CreatePrivDir}. 614 615check_and_install_configfiles( 616 Configs, LogDir, #opts{ 617 event_handlers = EvHandlers, 618 ct_hooks = CTHooks, 619 enable_builtin_hooks = EnableBuiltinHooks} ) -> 620 case ct_config:check_config_files(Configs) of 621 false -> 622 install([{config,Configs}, 623 {event_handler,EvHandlers}, 624 {ct_hooks,CTHooks}, 625 {enable_builtin_hooks,EnableBuiltinHooks}], LogDir); 626 {value,{error,{nofile,File}}} -> 627 {error,{cant_read_config_file,File}}; 628 {value,{error,{wrong_config,Message}}}-> 629 {error,{wrong_config,Message}}; 630 {value,{error,{callback,Info}}} -> 631 {error,{cant_load_callback_module,Info}} 632 end. 633 634script_start3(Opts, Args) -> 635 Opts1 = get_start_opt(step, 636 fun(Step) -> 637 Opts#opts{step = Step, 638 cover = undefined} 639 end, Opts, Args), 640 case {proplists:get_value(dir, Args), 641 proplists:get_value(suite, Args), 642 groups_and_cases(proplists:get_value(group, Args), 643 proplists:get_value(testcase, Args))} of 644 %% flag specified without data 645 {_,_,Error={error,_}} -> 646 Error; 647 {_,[],_} -> 648 {error,no_suite_specified}; 649 {[],_,_} -> 650 {error,no_dir_specified}; 651 652 {Dirs,undefined,[]} when is_list(Dirs) -> 653 script_start4(Opts#opts{tests = tests(Dirs)}, Args); 654 655 {undefined,Suites,[]} when is_list(Suites) -> 656 Ts = tests([suite_to_test(S) || S <- Suites]), 657 script_start4(Opts1#opts{tests = Ts}, Args); 658 659 {undefined,Suite,GsAndCs} when is_list(Suite) -> 660 case [suite_to_test(S) || S <- Suite] of 661 DirMods = [_] -> 662 Ts = tests(DirMods, GsAndCs), 663 script_start4(Opts1#opts{tests = Ts}, Args); 664 [_,_|_] -> 665 {error,multiple_suites_and_cases}; 666 _ -> 667 {error,incorrect_start_options} 668 end; 669 670 {[_,_|_],Suite,[]} when is_list(Suite) -> 671 {error,multiple_dirs_and_suites}; 672 673 {[Dir],Suite,GsAndCs} when is_list(Dir), is_list(Suite) -> 674 case [suite_to_test(Dir,S) || S <- Suite] of 675 DirMods when GsAndCs == [] -> 676 Ts = tests(DirMods), 677 script_start4(Opts1#opts{tests = Ts}, Args); 678 DirMods = [_] when GsAndCs /= [] -> 679 Ts = tests(DirMods, GsAndCs), 680 script_start4(Opts1#opts{tests = Ts}, Args); 681 [_,_|_] when GsAndCs /= [] -> 682 {error,multiple_suites_and_cases}; 683 _ -> 684 {error,incorrect_start_options} 685 end; 686 687 {undefined,undefined,GsAndCs} when GsAndCs /= [] -> 688 {error,incorrect_start_options}; 689 690 {undefined,undefined,_} -> 691 if Opts#opts.shell -> 692 script_start4(Opts#opts{tests = []}, Args); 693 true -> 694 %% no start options, use default "-dir ./" 695 {ok,Dir} = file:get_cwd(), 696 io:format("ct_run -dir ~ts~n~n", [Dir]), 697 script_start4(Opts#opts{tests = tests([Dir])}, Args) 698 end 699 end. 700 701script_start4(#opts{label = Label, profile = Profile, 702 shell = true, config = Config, 703 event_handlers = EvHandlers, 704 ct_hooks = CTHooks, 705 logopts = LogOpts, 706 verbosity = Verbosity, 707 enable_builtin_hooks = EnableBuiltinHooks, 708 logdir = LogDir, testspec_files = Specs}, _Args) -> 709 710 %% label - used by ct_logs 711 application:set_env(common_test, test_label, Label), 712 713 %% profile - used in ct_util 714 application:set_env(common_test, profile, Profile), 715 716 if Config == [] -> 717 ok; 718 true -> 719 io:format("\nInstalling: ~tp\n\n", [Config]) 720 end, 721 case install([{config,Config},{event_handler,EvHandlers}, 722 {ct_hooks, CTHooks}, 723 {enable_builtin_hooks,EnableBuiltinHooks}]) of 724 ok -> 725 _ = ct_util:start(interactive, LogDir, 726 add_verbosity_defaults(Verbosity)), 727 ct_util:set_testdata({logopts, LogOpts}), 728 log_ts_names(Specs), 729 io:nl(), 730 interactive_mode; 731 Error -> 732 Error 733 end; 734script_start4(Opts = #opts{tests = Tests}, Args) -> 735 do_run(Tests, [], Opts, Args). 736 737script_usage() -> 738 io:format("\nUsage:\n\n"), 739 io:format("Run tests from command line:\n\n" 740 "\tct_run -dir TestDir1 TestDir2 .. TestDirN |" 741 "\n\t [-dir TestDir] -suite Suite1 Suite2 .. SuiteN" 742 "\n\t [-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]" 743 "\n\t [-step [config | keep_inactive]]" 744 "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" 745 "\n\t [-userconfig CallbackModule ConfigFile1 .. ConfigFileN]" 746 "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]" 747 "\n\t [-logdir LogDir]" 748 "\n\t [-logopts LogOpt1 LogOpt2 .. LogOptN]" 749 "\n\t [-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" 750 "\n\t [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" 751 "\n\t [-stylesheet CSSFile]" 752 "\n\t [-cover CoverCfgFile]" 753 "\n\t [-cover_stop Bool]" 754 "\n\t [-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" 755 "\n\t [-ct_hooks CTHook1 CTHook2 .. CTHookN]" 756 "\n\t [-include InclDir1 InclDir2 .. InclDirN]" 757 "\n\t [-no_auto_compile]" 758 "\n\t [-abort_if_missing_suites]" 759 "\n\t [-multiply_timetraps N]" 760 "\n\t [-scale_timetraps]" 761 "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" 762 "\n\t [-basic_html]" 763 "\n\t [-no_esc_chars]" 764 "\n\t [-repeat N] |" 765 "\n\t [-duration HHMMSS [-force_stop [skip_rest]]] |" 766 "\n\t [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]" 767 "\n\t [-exit_status ignore_config]" 768 "\n\t [-help]\n\n"), 769 io:format("Run tests using test specification:\n\n" 770 "\tct_run -spec TestSpec1 TestSpec2 .. TestSpecN" 771 "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" 772 "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]" 773 "\n\t [-logdir LogDir]" 774 "\n\t [-logopts LogOpt1 LogOpt2 .. LogOptN]" 775 "\n\t [-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" 776 "\n\t [-allow_user_terms]" 777 "\n\t [-join_specs]" 778 "\n\t [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" 779 "\n\t [-stylesheet CSSFile]" 780 "\n\t [-cover CoverCfgFile]" 781 "\n\t [-cover_stop Bool]" 782 "\n\t [-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" 783 "\n\t [-ct_hooks CTHook1 CTHook2 .. CTHookN]" 784 "\n\t [-include InclDir1 InclDir2 .. InclDirN]" 785 "\n\t [-no_auto_compile]" 786 "\n\t [-abort_if_missing_suites]" 787 "\n\t [-multiply_timetraps N]" 788 "\n\t [-scale_timetraps]" 789 "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" 790 "\n\t [-basic_html]" 791 "\n\t [-no_esc_chars]" 792 "\n\t [-repeat N] |" 793 "\n\t [-duration HHMMSS [-force_stop [skip_rest]]] |" 794 "\n\t [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]]\n\n"), 795 io:format("Refresh the HTML index files:\n\n" 796 "\tct_run -refresh_logs [LogDir]" 797 " [-logdir LogDir] " 798 " [-basic_html]\n\n"), 799 io:format("Run CT in interactive mode:\n\n" 800 "\tct_run -shell" 801 "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" 802 "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]\n\n"), 803 io:format("Run tests in web based GUI:\n\n" 804 "\n\t [-config ConfigFile1 ConfigFile2 .. ConfigFileN]" 805 "\n\t [-decrypt_key Key] | [-decrypt_file KeyFile]" 806 "\n\t [-dir TestDir1 TestDir2 .. TestDirN] |" 807 "\n\t [-suite Suite [-case Case]]" 808 "\n\t [-logopts LogOpt1 LogOpt2 .. LogOptN]" 809 "\n\t [-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" 810 "\n\t [-include InclDir1 InclDir2 .. InclDirN]" 811 "\n\t [-no_auto_compile]" 812 "\n\t [-abort_if_missing_suites]" 813 "\n\t [-multiply_timetraps N]" 814 "\n\t [-scale_timetraps]" 815 "\n\t [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc]" 816 "\n\t [-basic_html]" 817 "\n\t [-no_esc_chars]\n\n"). 818 819install(Opts) -> 820 install(Opts, "."). 821 822install(Opts, LogDir) -> 823 824 ConfOpts = ct_config:add_default_callback(Opts), 825 826 case application:get_env(common_test, decrypt) of 827 {ok,_} -> 828 ok; 829 _ -> 830 case lists:keysearch(decrypt, 1, Opts) of 831 {value,{_,KeyOrFile}} -> 832 application:set_env(common_test, decrypt, KeyOrFile); 833 false -> 834 application:unset_env(common_test, decrypt) 835 end 836 end, 837 case whereis(ct_util_server) of 838 undefined -> 839 VarFile = variables_file_name(LogDir), 840 case file:open(VarFile, [write, {encoding,utf8}]) of 841 {ok,Fd} -> 842 _ = [io:format(Fd, "~tp.\n", [Opt]) || Opt <- ConfOpts], 843 ok = file:close(Fd); 844 {error,Reason} -> 845 io:format("CT failed to install configuration data. Please " 846 "verify that the log directory exists and that " 847 "write permission is set.\n\n", []), 848 {error,{VarFile,Reason}} 849 end; 850 _ -> 851 io:format("It is not possible to install CT while running " 852 "in interactive mode.\n" 853 "To exit this mode, run ct:stop_interactive().\n" 854 "To enter the interactive mode again, " 855 "run ct:start_interactive()\n\n", []), 856 {error,interactive_mode} 857 end. 858 859variables_file_name(Dir) -> 860 filename:join(Dir, "variables-"++atom_to_list(node())). 861 862run_test(StartOpt) when is_tuple(StartOpt) -> 863 run_test([StartOpt]); 864 865run_test(StartOpts) when is_list(StartOpts) -> 866 CTPid = spawn(run_test1_fun(StartOpts)), 867 Ref = monitor(process, CTPid), 868 receive 869 {'DOWN',Ref,process,CTPid,{user_error,Error}} -> 870 {error,Error}; 871 {'DOWN',Ref,process,CTPid,Other} -> 872 Other 873 end. 874 875-spec run_test1_fun(_) -> fun(() -> no_return()). 876 877run_test1_fun(StartOpts) -> 878 fun() -> 879 ct_util:mark_process(), 880 run_test1(StartOpts) 881 end. 882 883run_test1(StartOpts) when is_list(StartOpts) -> 884 case proplists:get_value(refresh_logs, StartOpts) of 885 undefined -> 886 Tracing = start_trace(StartOpts), 887 {ok,Cwd} = file:get_cwd(), 888 io:format("~nCommon Test starting (cwd is ~ts)~n~n", [Cwd]), 889 Res = 890 case ct_repeat:loop_test(func, StartOpts) of 891 false -> 892 case catch run_test2(StartOpts) of 893 {'EXIT',Reason} -> 894 ok = file:set_cwd(Cwd), 895 {error,Reason}; 896 Result -> 897 Result 898 end; 899 Result -> 900 Result 901 end, 902 stop_trace(Tracing), 903 exit(Res); 904 RefreshDir -> 905 %% log_cleanup - used by ct_logs 906 KeepLogs = get_start_opt(keep_logs, 907 fun ct_logs:parse_keep_logs/1, 908 all, 909 StartOpts), 910 application:set_env(common_test, keep_logs, KeepLogs), 911 ok = refresh_logs(?abs(RefreshDir)), 912 exit(done) 913 end. 914 915run_test2(StartOpts) -> 916 %% label 917 Label = get_start_opt(label, fun(Lbl) when is_list(Lbl) -> Lbl; 918 (Lbl) when is_atom(Lbl) -> atom_to_list(Lbl) 919 end, StartOpts), 920 %% profile 921 Profile = get_start_opt(profile, fun(Prof) when is_list(Prof) -> 922 Prof; 923 (Prof) when is_atom(Prof) -> 924 atom_to_list(Prof) 925 end, StartOpts), 926 %% logdir 927 LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end, 928 StartOpts), 929 %% logopts 930 LogOpts = get_start_opt(logopts, value, [], StartOpts), 931 932 %% verbosity 933 Verbosity = 934 get_start_opt(verbosity, 935 fun(VLvls) when is_list(VLvls) -> 936 lists:map(fun(VLvl = {_Cat,_Lvl}) -> 937 VLvl; 938 (Lvl) -> 939 {'$unspecified',Lvl} 940 end, VLvls); 941 (VLvl) when is_integer(VLvl) -> 942 [{'$unspecified',VLvl}] 943 end, [], StartOpts), 944 945 %% config & userconfig 946 CfgFiles = ct_config:get_config_file_list(StartOpts), 947 948 %% event handlers 949 EvHandlers = 950 case proplists:get_value(event_handler, StartOpts) of 951 undefined -> 952 []; 953 H when is_atom(H) -> 954 [{H,[]}]; 955 H -> 956 Hs = 957 if is_tuple(H) -> [H]; 958 is_list(H) -> H; 959 true -> [] 960 end, 961 lists:flatten( 962 lists:map(fun(EH) when is_atom(EH) -> 963 {EH,[]}; 964 ({HL,Args}) when is_list(HL) -> 965 [{EH,Args} || EH <- HL]; 966 ({EH,Args}) when is_atom(EH) -> 967 {EH,Args}; 968 (_) -> 969 [] 970 end, Hs)) 971 end, 972 973 %% CT Hooks 974 CTHooks = get_start_opt(ct_hooks, value, [], StartOpts), 975 EnableBuiltinHooks = get_start_opt(enable_builtin_hooks, 976 fun(EBH) when EBH == true; 977 EBH == false -> 978 EBH 979 end, undefined, StartOpts), 980 981 %% silent connections 982 SilentConns = get_start_opt(silent_connections, 983 fun(all) -> [all]; 984 (Conns) -> Conns 985 end, [], StartOpts), 986 %% stylesheet 987 Stylesheet = get_start_opt(stylesheet, 988 fun(SS) -> ?abs(SS) end, 989 StartOpts), 990 %% code coverage 991 Cover = get_start_opt(cover, 992 fun(CoverFile) -> ?abs(CoverFile) end, StartOpts), 993 CoverStop = get_start_opt(cover_stop, value, StartOpts), 994 995 %% timetrap manipulation 996 MultiplyTT = get_start_opt(multiply_timetraps, value, StartOpts), 997 ScaleTT = get_start_opt(scale_timetraps, value, StartOpts), 998 999 %% create unique priv dir names 1000 CreatePrivDir = get_start_opt(create_priv_dir, value, StartOpts), 1001 1002 %% auto compile & include files 1003 {AutoCompile,Include} = 1004 case proplists:get_value(auto_compile, StartOpts) of 1005 undefined -> 1006 application:set_env(common_test, auto_compile, true), 1007 InclDirs = 1008 case proplists:get_value(include, StartOpts) of 1009 undefined -> 1010 []; 1011 Incls when is_list(hd(Incls)) -> 1012 [filename:absname(IDir) || IDir <- Incls]; 1013 Incl when is_list(Incl) -> 1014 [filename:absname(Incl)] 1015 end, 1016 case os:getenv("CT_INCLUDE_PATH") of 1017 false -> 1018 application:set_env(common_test, include, InclDirs), 1019 {undefined,InclDirs}; 1020 CtInclPath -> 1021 InclDirs1 = string:lexemes(CtInclPath, [$:,$ ,$,]), 1022 AllInclDirs = InclDirs1++InclDirs, 1023 application:set_env(common_test, include, AllInclDirs), 1024 {undefined,AllInclDirs} 1025 end; 1026 ACBool -> 1027 application:set_env(common_test, auto_compile, ACBool), 1028 {ACBool,[]} 1029 end, 1030 1031 %% abort test run if some suites can't be compiled 1032 AbortIfMissing = get_start_opt(abort_if_missing_suites, value, false, 1033 StartOpts), 1034 1035 %% decrypt config file 1036 case proplists:get_value(decrypt, StartOpts) of 1037 undefined -> 1038 application:unset_env(common_test, decrypt); 1039 Key={key,_} -> 1040 application:set_env(common_test, decrypt, Key); 1041 {file,KeyFile} -> 1042 application:set_env(common_test, decrypt, {file,?abs(KeyFile)}) 1043 end, 1044 1045 %% basic html - used by ct_logs 1046 BasicHtml = 1047 case proplists:get_value(basic_html, StartOpts) of 1048 undefined -> 1049 application:set_env(common_test, basic_html, false), 1050 undefined; 1051 BasicHtmlBool -> 1052 application:set_env(common_test, basic_html, BasicHtmlBool), 1053 BasicHtmlBool 1054 end, 1055 %% esc_chars - used by ct_logs 1056 EscChars = 1057 case proplists:get_value(esc_chars, StartOpts) of 1058 undefined -> 1059 application:set_env(common_test, esc_chars, true), 1060 undefined; 1061 EscCharsBool -> 1062 application:set_env(common_test, esc_chars, EscCharsBool), 1063 EscCharsBool 1064 end, 1065 %% disable_log_cache - used by ct_logs 1066 case proplists:get_value(disable_log_cache, StartOpts) of 1067 undefined -> 1068 application:set_env(common_test, disable_log_cache, false); 1069 DisableCacheBool -> 1070 application:set_env(common_test, disable_log_cache, DisableCacheBool) 1071 end, 1072 %% log_cleanup - used by ct_logs 1073 KeepLogs = get_start_opt(keep_logs, 1074 fun ct_logs:parse_keep_logs/1, 1075 all, 1076 StartOpts), 1077 application:set_env(common_test, keep_logs, KeepLogs), 1078 1079 %% stepped execution 1080 Step = get_start_opt(step, value, StartOpts), 1081 1082 Opts = #opts{label = Label, profile = Profile, 1083 cover = Cover, cover_stop = CoverStop, 1084 step = Step, logdir = LogDir, 1085 logopts = LogOpts, basic_html = BasicHtml, 1086 esc_chars = EscChars, 1087 config = CfgFiles, 1088 verbosity = Verbosity, 1089 event_handlers = EvHandlers, 1090 ct_hooks = CTHooks, 1091 enable_builtin_hooks = EnableBuiltinHooks, 1092 auto_compile = AutoCompile, 1093 abort_if_missing_suites = AbortIfMissing, 1094 include = Include, 1095 silent_connections = SilentConns, 1096 stylesheet = Stylesheet, 1097 multiply_timetraps = MultiplyTT, 1098 scale_timetraps = ScaleTT, 1099 create_priv_dir = CreatePrivDir, 1100 starter = ct}, 1101 1102 %% test specification 1103 case proplists:get_value(spec, StartOpts) of 1104 undefined -> 1105 case lists:keysearch(prepared_tests, 1, StartOpts) of 1106 {value,{_,{Run,Skip},Specs}} -> % use prepared tests 1107 run_prepared(Run, Skip, Opts#opts{testspec_files = Specs}, 1108 StartOpts); 1109 false -> 1110 run_dir(Opts, StartOpts) 1111 end; 1112 Specs -> 1113 Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts), 1114 %% using testspec(s) as input for test 1115 run_spec_file(Relaxed, Opts#opts{testspec_files = Specs}, StartOpts) 1116 end. 1117 1118run_spec_file(Relaxed, 1119 Opts = #opts{testspec_files = Specs}, 1120 StartOpts) -> 1121 Specs1 = case Specs of 1122 [X|_] when is_integer(X) -> [Specs]; 1123 _ -> Specs 1124 end, 1125 AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), 1126 AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts), 1127 try ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of 1128 TestSpecData -> 1129 run_all_specs(TestSpecData, Opts, StartOpts, []) 1130 catch 1131 throw:{error,CTReason}:StackTrace -> 1132 exit({error,{invalid_testspec,{CTReason,StackTrace}}}); 1133 _:CTReason:StackTrace -> 1134 exit({error,{invalid_testspec,{CTReason,StackTrace}}}) 1135 end. 1136 1137run_all_specs([], _, _, TotResult) -> 1138 TotResult1 = lists:reverse(TotResult), 1139 case lists:keysearch('EXIT', 1, TotResult1) of 1140 {value,{_,_,ExitReason}} -> 1141 exit(ExitReason); 1142 false -> 1143 case lists:keysearch(error, 1, TotResult1) of 1144 {value,Error} -> 1145 Error; 1146 false -> 1147 lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, 1148 {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> 1149 {Ok1+Ok,Fail1+Fail, 1150 {UserSkip1+UserSkip, 1151 AutoSkip1+AutoSkip}} 1152 end, {0,0,{0,0}}, TotResult1) 1153 end 1154 end; 1155 1156run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) -> 1157 Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts), 1158 AllConfig = merge_vals([Opts#opts.config, TSConfig]), 1159 try run_one_spec(TS, 1160 Combined#opts{config = AllConfig, 1161 current_testspec=TS}, 1162 StartOpts) of 1163 Result -> 1164 run_all_specs(TSs, Opts, StartOpts, [Result | TotResult]) 1165 catch 1166 _ : Reason -> 1167 run_all_specs(TSs, Opts, StartOpts, [{error,Reason} | TotResult]) 1168 end. 1169 1170run_one_spec(TS, CombinedOpts, StartOpts) -> 1171 #opts{logdir = Logdir, config = Config} = CombinedOpts, 1172 case check_and_install_configfiles(Config, Logdir, CombinedOpts) of 1173 ok -> 1174 {Run,Skip} = ct_testspec:prepare_tests(TS, node()), 1175 reformat_result(catch do_run(Run, Skip, CombinedOpts, StartOpts)); 1176 Error -> 1177 Error 1178 end. 1179 1180run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, 1181 config = CfgFiles}, 1182 StartOpts) -> 1183 LogDir1 = which(logdir, LogDir), 1184 case check_and_install_configfiles(CfgFiles, LogDir1, Opts) of 1185 ok -> 1186 reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1}, 1187 StartOpts)); 1188 {error,_Reason} = Error -> 1189 exit(Error) 1190 end. 1191 1192check_config_file(Callback, File)-> 1193 case code:is_loaded(Callback) of 1194 false -> 1195 case code:load_file(Callback) of 1196 {module,_} -> ok; 1197 {error,Why} -> exit({error,{cant_load_callback_module,Why}}) 1198 end; 1199 _ -> 1200 ok 1201 end, 1202 case Callback:check_parameter(File) of 1203 {ok,{file,File}}-> 1204 ?abs(File); 1205 {ok,{config,_}}-> 1206 File; 1207 {error,{wrong_config,Message}}-> 1208 exit({error,{wrong_config,{Callback,Message}}}); 1209 {error,{nofile,File}}-> 1210 exit({error,{no_such_file,?abs(File)}}) 1211 end. 1212 1213run_dir(Opts = #opts{logdir = LogDir, 1214 config = CfgFiles, 1215 event_handlers = EvHandlers, 1216 ct_hooks = CTHook, 1217 enable_builtin_hooks = EnableBuiltinHooks}, 1218 StartOpts) -> 1219 LogDir1 = which(logdir, LogDir), 1220 Opts1 = Opts#opts{logdir = LogDir1}, 1221 AbsCfgFiles = 1222 lists:map(fun({Callback,FileList})-> 1223 case code:is_loaded(Callback) of 1224 {file,_Path}-> 1225 ok; 1226 false -> 1227 case code:load_file(Callback) of 1228 {module,Callback}-> 1229 ok; 1230 {error,_}-> 1231 exit({error,{no_such_module, 1232 Callback}}) 1233 end 1234 end, 1235 {Callback, 1236 lists:map(fun(File)-> 1237 check_config_file(Callback, File) 1238 end, FileList)} 1239 end, CfgFiles), 1240 case install([{config,AbsCfgFiles}, 1241 {event_handler,EvHandlers}, 1242 {ct_hooks, CTHook}, 1243 {enable_builtin_hooks,EnableBuiltinHooks}], LogDir1) of 1244 ok -> ok; 1245 {error,_IReason} = IError -> exit(IError) 1246 end, 1247 case {proplists:get_value(dir, StartOpts), 1248 proplists:get_value(suite, StartOpts), 1249 groups_and_cases(proplists:get_value(group, StartOpts), 1250 proplists:get_value(testcase, StartOpts))} of 1251 %% flag specified without data 1252 {_,_,Error={error,_}} -> 1253 Error; 1254 {_,[],_} -> 1255 {error,no_suite_specified}; 1256 {[],_,_} -> 1257 {error,no_dir_specified}; 1258 1259 {Dirs=[Hd|_],undefined,[]} when is_list(Dirs), not is_integer(Hd) -> 1260 Dirs1 = [if is_atom(D) -> atom_to_list(D); 1261 true -> D end || D <- Dirs], 1262 reformat_result(catch do_run(tests(Dirs1), [], Opts1, StartOpts)); 1263 1264 {Dir=[Hd|_],undefined,[]} when is_list(Dir) and is_integer(Hd) -> 1265 reformat_result(catch do_run(tests(Dir), [], Opts1, StartOpts)); 1266 1267 {Dir,undefined,[]} when is_atom(Dir) and (Dir /= undefined) -> 1268 reformat_result(catch do_run(tests(atom_to_list(Dir)), 1269 [], Opts1, StartOpts)); 1270 1271 {undefined,Suites=[Hd|_],[]} when not is_integer(Hd) -> 1272 Suites1 = [suite_to_test(S) || S <- Suites], 1273 reformat_result(catch do_run(tests(Suites1), [], Opts1, StartOpts)); 1274 1275 {undefined,Suite,[]} when is_atom(Suite) and 1276 (Suite /= undefined) -> 1277 {Dir,Mod} = suite_to_test(Suite), 1278 reformat_result(catch do_run(tests(Dir, Mod), [], Opts1, StartOpts)); 1279 1280 {undefined,Suite,GsAndCs} when is_atom(Suite) and 1281 (Suite /= undefined) -> 1282 {Dir,Mod} = suite_to_test(Suite), 1283 reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), 1284 [], Opts1, StartOpts)); 1285 1286 {undefined,[Hd,_|_],_GsAndCs} when not is_integer(Hd) -> 1287 exit({error,multiple_suites_and_cases}); 1288 1289 {undefined,Suite=[Hd|Tl],GsAndCs} when is_integer(Hd) ; 1290 (is_list(Hd) and (Tl == [])) ; 1291 (is_atom(Hd) and (Tl == [])) -> 1292 {Dir,Mod} = suite_to_test(Suite), 1293 reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), 1294 [], Opts1, StartOpts)); 1295 1296 {[Hd,_|_],_Suites,[]} when is_list(Hd) ; not is_integer(Hd) -> 1297 exit({error,multiple_dirs_and_suites}); 1298 1299 {undefined,undefined,GsAndCs} when GsAndCs /= [] -> 1300 exit({error,incorrect_start_options}); 1301 1302 {Dir,Suite,GsAndCs} when is_integer(hd(Dir)) ; 1303 (is_atom(Dir) and (Dir /= undefined)) ; 1304 ((length(Dir) == 1) and is_atom(hd(Dir))) ; 1305 ((length(Dir) == 1) and is_list(hd(Dir))) -> 1306 Dir1 = if is_atom(Dir) -> atom_to_list(Dir); 1307 true -> Dir end, 1308 if Suite == undefined -> 1309 exit({error,incorrect_start_options}); 1310 1311 is_integer(hd(Suite)) ; 1312 (is_atom(Suite) and (Suite /= undefined)) ; 1313 ((length(Suite) == 1) and is_atom(hd(Suite))) ; 1314 ((length(Suite) == 1) and is_list(hd(Suite))) -> 1315 {Dir2,Mod} = suite_to_test(Dir1, Suite), 1316 case GsAndCs of 1317 [] -> 1318 reformat_result(catch do_run(tests(Dir2, Mod), 1319 [], Opts1, StartOpts)); 1320 _ -> 1321 reformat_result(catch do_run(tests(Dir2, Mod, 1322 GsAndCs), 1323 [], Opts1, StartOpts)) 1324 end; 1325 1326 is_list(Suite) -> % multiple suites 1327 case [suite_to_test(Dir1, S) || S <- Suite] of 1328 [_,_|_] when GsAndCs /= [] -> 1329 exit({error,multiple_suites_and_cases}); 1330 [{Dir2,Mod}] when GsAndCs /= [] -> 1331 reformat_result(catch do_run(tests(Dir2, Mod, 1332 GsAndCs), 1333 [], Opts1, StartOpts)); 1334 DirMods -> 1335 reformat_result(catch do_run(tests(DirMods), 1336 [], Opts1, StartOpts)) 1337 end 1338 end; 1339 1340 {undefined,undefined,[]} -> 1341 {ok,Dir} = file:get_cwd(), 1342 %% No start options, use default {dir,CWD} 1343 reformat_result(catch do_run(tests(Dir), [], Opts1, StartOpts)); 1344 1345 {Dir,Suite,GsAndCs} -> 1346 exit({error,{incorrect_start_options,{Dir,Suite,GsAndCs}}}) 1347 end. 1348 1349run_testspec(TestSpec) -> 1350 CTPid = spawn(run_testspec1_fun(TestSpec)), 1351 Ref = monitor(process, CTPid), 1352 receive 1353 {'DOWN',Ref,process,CTPid,{user_error,Error}} -> 1354 Error; 1355 {'DOWN',Ref,process,CTPid,Other} -> 1356 Other 1357 end. 1358 1359-spec run_testspec1_fun(_) -> fun(() -> no_return()). 1360 1361run_testspec1_fun(TestSpec) -> 1362 fun() -> 1363 ct_util:mark_process(), 1364 run_testspec1(TestSpec) 1365 end. 1366 1367run_testspec1(TestSpec) -> 1368 {ok,Cwd} = file:get_cwd(), 1369 io:format("~nCommon Test starting (cwd is ~ts)~n~n", [Cwd]), 1370 case catch run_testspec2(TestSpec) of 1371 {'EXIT',Reason} -> 1372 ok = file:set_cwd(Cwd), 1373 exit({error,Reason}); 1374 Result -> 1375 exit(Result) 1376 end. 1377 1378run_testspec2(File) when is_list(File), is_integer(hd(File)) -> 1379 case file:read_file_info(File) of 1380 {ok,_} -> 1381 exit("Bad argument, " 1382 "use ct:run_test([{spec," ++ File ++ "}])"); 1383 _ -> 1384 exit("Bad argument, list of tuples expected, " 1385 "use ct:run_test/1 for test specification files") 1386 end; 1387 1388run_testspec2(TestSpec) -> 1389 case catch ct_testspec:collect_tests_from_list(TestSpec, false) of 1390 {E,CTReason} when E == error ; E == 'EXIT' -> 1391 exit({error,CTReason}); 1392 TS -> 1393 Opts = get_data_for_node(TS, node()), 1394 1395 AllInclude = 1396 case os:getenv("CT_INCLUDE_PATH") of 1397 false -> 1398 Opts#opts.include; 1399 CtInclPath -> 1400 EnvInclude = string:lexemes(CtInclPath, [$:,$ ,$,]), 1401 EnvInclude++Opts#opts.include 1402 end, 1403 application:set_env(common_test, include, AllInclude), 1404 1405 LogDir1 = which(logdir,Opts#opts.logdir), 1406 case check_and_install_configfiles( 1407 Opts#opts.config, LogDir1, Opts) of 1408 ok -> 1409 Opts1 = Opts#opts{testspec_files = [], 1410 logdir = LogDir1, 1411 include = AllInclude}, 1412 {Run,Skip} = ct_testspec:prepare_tests(TS, node()), 1413 reformat_result(catch do_run(Run, Skip, Opts1, [])); 1414 {error,_GCFReason} = GCFError -> 1415 exit(GCFError) 1416 end 1417 end. 1418 1419get_data_for_node(#testspec{label = Labels, 1420 profile = Profiles, 1421 logdir = LogDirs, 1422 logopts = LogOptsList, 1423 basic_html = BHs, 1424 esc_chars = EscChs, 1425 stylesheet = SSs, 1426 verbosity = VLvls, 1427 silent_connections = SilentConnsList, 1428 cover = CoverFs, 1429 cover_stop = CoverStops, 1430 config = Cfgs, 1431 userconfig = UsrCfgs, 1432 event_handler = EvHs, 1433 ct_hooks = CTHooks, 1434 enable_builtin_hooks = EnableBuiltinHooks, 1435 auto_compile = ACs, 1436 abort_if_missing_suites = AiMSs, 1437 include = Incl, 1438 multiply_timetraps = MTs, 1439 scale_timetraps = STs, 1440 create_priv_dir = PDs}, Node) -> 1441 Label = proplists:get_value(Node, Labels), 1442 Profile = proplists:get_value(Node, Profiles), 1443 LogDir = case proplists:get_value(Node, LogDirs) of 1444 undefined -> "."; 1445 Dir -> Dir 1446 end, 1447 LogOpts = case proplists:get_value(Node, LogOptsList) of 1448 undefined -> []; 1449 LOs -> LOs 1450 end, 1451 BasicHtml = proplists:get_value(Node, BHs), 1452 EscChars = proplists:get_value(Node, EscChs), 1453 Stylesheet = proplists:get_value(Node, SSs), 1454 Verbosity = case proplists:get_value(Node, VLvls) of 1455 undefined -> []; 1456 Lvls -> Lvls 1457 end, 1458 SilentConns = case proplists:get_value(Node, SilentConnsList) of 1459 undefined -> []; 1460 SCs -> SCs 1461 end, 1462 Cover = proplists:get_value(Node, CoverFs), 1463 CoverStop = proplists:get_value(Node, CoverStops), 1464 MT = proplists:get_value(Node, MTs), 1465 ST = proplists:get_value(Node, STs), 1466 CreatePrivDir = proplists:get_value(Node, PDs), 1467 ConfigFiles = [{?ct_config_txt,F} || {N,F} <- Cfgs, N==Node] ++ 1468 [CBF || {N,CBF} <- UsrCfgs, N==Node], 1469 EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node], 1470 FiltCTHooks = [Hook || {N,Hook} <- CTHooks, N==Node], 1471 AutoCompile = proplists:get_value(Node, ACs), 1472 AbortIfMissing = proplists:get_value(Node, AiMSs), 1473 Include = [I || {N,I} <- Incl, N==Node], 1474 #opts{label = Label, 1475 profile = Profile, 1476 logdir = LogDir, 1477 logopts = LogOpts, 1478 basic_html = BasicHtml, 1479 esc_chars = EscChars, 1480 stylesheet = Stylesheet, 1481 verbosity = Verbosity, 1482 silent_connections = SilentConns, 1483 cover = Cover, 1484 cover_stop = CoverStop, 1485 config = ConfigFiles, 1486 event_handlers = EvHandlers, 1487 ct_hooks = FiltCTHooks, 1488 enable_builtin_hooks = EnableBuiltinHooks, 1489 auto_compile = AutoCompile, 1490 abort_if_missing_suites = AbortIfMissing, 1491 include = Include, 1492 multiply_timetraps = MT, 1493 scale_timetraps = ST, 1494 create_priv_dir = CreatePrivDir}. 1495 1496refresh_logs(LogDir) -> 1497 {ok,Cwd} = file:get_cwd(), 1498 case file:set_cwd(LogDir) of 1499 E = {error,_Reason} -> 1500 E; 1501 _ -> 1502 case catch ct_logs:make_all_suites_index(refresh) of 1503 {'EXIT',ASReason} -> 1504 ok = file:set_cwd(Cwd), 1505 {error,{all_suites_index,ASReason}}; 1506 _ -> 1507 case catch ct_logs:make_all_runs_index(refresh) of 1508 {'EXIT',ARReason} -> 1509 ok = file:set_cwd(Cwd), 1510 {error,{all_runs_index,ARReason}}; 1511 _ -> 1512 ok = file:set_cwd(Cwd), 1513 io:format("Logs in ~ts refreshed!~n",[LogDir]), 1514 ok 1515 end 1516 end 1517 end. 1518 1519which(logdir, undefined) -> 1520 "."; 1521which(logdir, Dir) -> 1522 Dir. 1523 1524choose_val(undefined, V1) -> 1525 V1; 1526choose_val(V0, _V1) -> 1527 V0. 1528 1529merge_vals(Vs) -> 1530 lists:append(Vs). 1531 1532merge_keyvals(Vs) -> 1533 make_unique(lists:append(Vs)). 1534 1535make_unique([Elem={Key,_} | Elems]) -> 1536 [Elem | make_unique(proplists:delete(Key, Elems))]; 1537make_unique([]) -> 1538 []. 1539 1540listify([C|_]=Str) when is_integer(C) -> [Str]; 1541listify(L) when is_list(L) -> L; 1542listify(E) -> [E]. 1543 1544delistify([E]) -> E; 1545delistify(E) -> E. 1546 1547 1548run(TestDir, Suite, Cases) -> 1549 case install([]) of 1550 ok -> 1551 reformat_result(catch do_run(tests(TestDir, Suite, Cases), [])); 1552 Error -> 1553 Error 1554 end. 1555 1556run(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) -> 1557 case install([]) of 1558 ok -> 1559 reformat_result(catch do_run(tests(TestDir, Suite), [])); 1560 Error -> 1561 Error 1562 end. 1563 1564run(TestDirs) -> 1565 case install([]) of 1566 ok -> 1567 reformat_result(catch do_run(tests(TestDirs), [])); 1568 Error -> 1569 Error 1570 end. 1571 1572reformat_result({'EXIT',{user_error,Reason}}) -> 1573 {error,Reason}; 1574reformat_result({user_error,Reason}) -> 1575 {error,Reason}; 1576reformat_result(Result) -> 1577 Result. 1578 1579suite_to_test(Suite) when is_atom(Suite) -> 1580 suite_to_test(atom_to_list(Suite)); 1581 1582suite_to_test(Suite) when is_list(Suite) -> 1583 {filename:dirname(Suite), 1584 list_to_atom(filename:rootname(filename:basename(Suite)))}. 1585 1586suite_to_test(Dir, Suite) when is_atom(Suite) -> 1587 suite_to_test(Dir, atom_to_list(Suite)); 1588 1589suite_to_test(Dir, Suite) when is_list(Suite) -> 1590 case filename:dirname(Suite) of 1591 "." -> 1592 {Dir,list_to_atom(filename:rootname(Suite))}; 1593 DirName -> % ignore Dir 1594 File = filename:basename(Suite), 1595 {DirName,list_to_atom(filename:rootname(File))} 1596 end. 1597 1598groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and 1599 ((Cs == undefined) or (Cs == [])) -> 1600 []; 1601groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] -> 1602 if (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> all; 1603 true -> [ensure_atom(C) || C <- listify(Cs)] 1604 end; 1605groups_and_cases(GOrGs, Cs) when (is_atom(GOrGs) orelse 1606 (is_list(GOrGs) andalso 1607 (is_atom(hd(GOrGs)) orelse 1608 (is_list(hd(GOrGs)) andalso 1609 is_atom(hd(hd(GOrGs))))))) -> 1610 if (Cs == undefined) or (Cs == []) or 1611 (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> 1612 [{GOrGs,all}]; 1613 true -> 1614 [{GOrGs,[ensure_atom(C) || C <- listify(Cs)]}] 1615 end; 1616groups_and_cases(Gs, Cs) when is_integer(hd(hd(Gs))) -> 1617 %% if list of strings, this comes from 'ct_run -group G1 G2 ...' and 1618 %% we need to parse the strings 1619 Gs1 = 1620 if (Gs == [all]) or (Gs == ["all"]) -> 1621 all; 1622 true -> 1623 lists:map(fun(G) -> 1624 {ok,Ts,_} = erl_scan:string(G++"."), 1625 {ok,Term} = erl_parse:parse_term(Ts), 1626 Term 1627 end, Gs) 1628 end, 1629 groups_and_cases(Gs1, Cs); 1630groups_and_cases(Gs, Cs) -> 1631 {error,{incorrect_group_or_case_option,Gs,Cs}}. 1632 1633tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) -> 1634 [{?testdir(TestDir,Suites),ensure_atom(Suites),all}]; 1635tests(TestDir, Suite, Cases) when is_list(TestDir), is_integer(hd(TestDir)) -> 1636 [{?testdir(TestDir,Suite),ensure_atom(Suite),Cases}]; 1637tests([TestDir], Suite, Cases) when is_list(TestDir), is_integer(hd(TestDir)) -> 1638 [{?testdir(TestDir,Suite),ensure_atom(Suite),Cases}]. 1639tests([{Dir,Suite}],Cases) -> 1640 [{?testdir(Dir,Suite),ensure_atom(Suite),Cases}]; 1641tests(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) -> 1642 tests(TestDir, ensure_atom(Suite), all); 1643tests([TestDir], Suite) when is_list(TestDir), is_integer(hd(TestDir)) -> 1644 tests(TestDir, ensure_atom(Suite), all). 1645tests(DirSuites) when is_list(DirSuites), is_tuple(hd(DirSuites)) -> 1646 [{?testdir(Dir,Suite),ensure_atom(Suite),all} || {Dir,Suite} <- DirSuites]; 1647tests(TestDir) when is_list(TestDir), is_integer(hd(TestDir)) -> 1648 tests([TestDir]); 1649tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) -> 1650 [{?testdir(TestDir,all),all,all} || TestDir <- TestDirs]. 1651 1652do_run(Tests, Misc) when is_list(Misc) -> 1653 do_run(Tests, Misc, ".", []). 1654 1655do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), 1656 is_list(LogDir), 1657 is_list(LogOpts) -> 1658 Opts = 1659 case proplists:get_value(step, Misc) of 1660 undefined -> 1661 #opts{}; 1662 StepOpts -> 1663 #opts{step = StepOpts} 1664 end, 1665 do_run(Tests, [], Opts#opts{logdir = LogDir}, []); 1666 1667do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> 1668 #opts{label = Label, profile = Profile, 1669 verbosity = VLvls} = Opts, 1670 %% label - used by ct_logs 1671 TestLabel = 1672 if Label == undefined -> undefined; 1673 is_atom(Label) -> atom_to_list(Label); 1674 is_list(Label) -> Label; 1675 true -> undefined 1676 end, 1677 application:set_env(common_test, test_label, TestLabel), 1678 1679 %% profile - used in ct_util 1680 TestProfile = 1681 if Profile == undefined -> undefined; 1682 is_atom(Profile) -> atom_to_list(Profile); 1683 is_list(Profile) -> Profile; 1684 true -> undefined 1685 end, 1686 application:set_env(common_test, profile, TestProfile), 1687 1688 case code:which(test_server) of 1689 non_existing -> 1690 {error,no_path_to_test_server}; 1691 _ -> 1692 %% This env variable is used by test_server to determine 1693 %% which framework it runs under. 1694 case os:getenv("TEST_SERVER_FRAMEWORK") of 1695 false -> 1696 os:putenv("TEST_SERVER_FRAMEWORK", "ct_framework"), 1697 os:putenv("TEST_SERVER_FRAMEWORK_NAME", "common_test"); 1698 "ct_framework" -> 1699 ok; 1700 Other -> 1701 erlang:display( 1702 list_to_atom( 1703 "Note: TEST_SERVER_FRAMEWORK = " ++ Other)) 1704 end, 1705 Verbosity = add_verbosity_defaults(VLvls), 1706 case ct_util:start(Opts#opts.logdir, Verbosity) of 1707 {error,interactive_mode} -> 1708 io:format("CT is started in interactive mode. " 1709 "To exit this mode, " 1710 "run ct:stop_interactive().\n" 1711 "To enter the interactive mode again, " 1712 "run ct:start_interactive()\n\n",[]), 1713 {error,interactive_mode}; 1714 _Pid -> 1715 ct_util:set_testdata({starter,Opts#opts.starter}), 1716 compile_and_run(Tests, Skip, 1717 Opts#opts{verbosity=Verbosity}, Args) 1718 end 1719 end. 1720 1721compile_and_run(Tests, Skip, Opts, Args) -> 1722 %% save stylesheet info 1723 ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), 1724 %% save logopts 1725 ct_util:set_testdata({logopts,Opts#opts.logopts}), 1726 %% save info about current testspec (testspec record or undefined) 1727 ct_util:set_testdata({testspec,Opts#opts.current_testspec}), 1728 1729 %% enable silent connections 1730 case Opts#opts.silent_connections of 1731 [] -> 1732 ok; 1733 Conns -> 1734 case lists:member(all, Conns) of 1735 true -> 1736 Conns1 = ct_util:override_silence_all_connections(), 1737 ct_logs:log("Silent connections", "~tp", [Conns1]); 1738 false -> 1739 ct_util:override_silence_connections(Conns), 1740 ct_logs:log("Silent connections", "~tp", [Conns]) 1741 end 1742 end, 1743 log_ts_names(Opts#opts.testspec_files), 1744 TestSuites = suite_tuples(Tests), 1745 1746 {_TestSuites1,SuiteMakeErrors,AllMakeErrors} = 1747 case application:get_env(common_test, auto_compile) of 1748 {ok,false} -> 1749 {TestSuites1,SuitesNotFound} = 1750 verify_suites(TestSuites), 1751 {TestSuites1,SuitesNotFound,SuitesNotFound}; 1752 _ -> 1753 {SuiteErrs,HelpErrs} = auto_compile(TestSuites), 1754 {TestSuites,SuiteErrs,SuiteErrs++HelpErrs} 1755 end, 1756 1757 case continue(AllMakeErrors, Opts#opts.abort_if_missing_suites) of 1758 true -> 1759 SavedErrors = save_make_errors(SuiteMakeErrors), 1760 ct_repeat:log_loop_info(Args), 1761 1762 try final_tests(Tests,Skip,SavedErrors) of 1763 {Tests1,Skip1} -> 1764 ReleaseSh = proplists:get_value(release_shell, Args), 1765 ct_util:set_testdata({release_shell,ReleaseSh}), 1766 TestResult = 1767 possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts), 1768 case TestResult of 1769 {Ok,Errors,Skipped} -> 1770 NoOfMakeErrors = 1771 lists:foldl(fun({_,BadMods}, X) -> 1772 X + length(BadMods) 1773 end, 0, SuiteMakeErrors), 1774 {Ok,Errors+NoOfMakeErrors,Skipped}; 1775 ErrorResult -> 1776 ErrorResult 1777 end 1778 catch 1779 _:BadFormat -> 1780 {error,BadFormat} 1781 end; 1782 false -> 1783 io:nl(), 1784 ct_util:stop(clean), 1785 BadMods = 1786 lists:foldr( 1787 fun({{_,_},Ms}, Acc) -> 1788 Ms ++ lists:foldl( 1789 fun(M, Acc1) -> 1790 lists:delete(M, Acc1) 1791 end, Acc, Ms) 1792 end, [], AllMakeErrors), 1793 {error,{make_failed,BadMods}} 1794 end. 1795 1796%% keep the shell as the top controlling process 1797possibly_spawn(false, Tests, Skip, Opts) -> 1798 TestResult = (catch do_run_test(Tests, Skip, Opts)), 1799 case TestResult of 1800 {EType,_} = Error when EType == user_error; 1801 EType == error -> 1802 ct_util:stop(clean), 1803 exit(Error); 1804 _ -> 1805 ct_util:stop(normal), 1806 TestResult 1807 end; 1808 1809%% we must return control to the shell now, so we spawn 1810%% a test supervisor process to keep an eye on the test run 1811possibly_spawn(true, Tests, Skip, Opts) -> 1812 CTUtilSrv = whereis(ct_util_server), 1813 Supervisor = 1814 fun() -> 1815 ct_util:mark_process(), 1816 process_flag(trap_exit, true), 1817 link(CTUtilSrv), 1818 TestRun = 1819 fun() -> 1820 ct_util:mark_process(), 1821 TestResult = (catch do_run_test(Tests, Skip, Opts)), 1822 case TestResult of 1823 {EType,_} = Error when EType == user_error; 1824 EType == error -> 1825 ct_util:stop(clean), 1826 exit(Error); 1827 _ -> 1828 ct_util:stop(normal), 1829 exit({ok,TestResult}) 1830 end 1831 end, 1832 TestRunPid = spawn_link(TestRun), 1833 receive 1834 {'EXIT',TestRunPid,{ok,TestResult}} -> 1835 io:format(user, "~nCommon Test returned ~tp~n~n", 1836 [TestResult]); 1837 {'EXIT',TestRunPid,Error} -> 1838 exit(Error) 1839 end 1840 end, 1841 unlink(CTUtilSrv), 1842 SupPid = spawn(Supervisor), 1843 io:format(user, "~nTest control handed over to process ~w~n~n", 1844 [SupPid]), 1845 SupPid. 1846 1847%% attempt to compile the modules specified in TestSuites 1848auto_compile(TestSuites) -> 1849 io:format("~nCommon Test: Running make in test directories...~n"), 1850 UserInclude = 1851 case application:get_env(common_test, include) of 1852 {ok,UserInclDirs} when length(UserInclDirs) > 0 -> 1853 io:format("Including the following directories:~n"), 1854 [begin io:format("~tp~n",[UserInclDir]), {i,UserInclDir} end || 1855 UserInclDir <- UserInclDirs]; 1856 _ -> 1857 [] 1858 end, 1859 SuiteMakeErrors = 1860 lists:flatmap(fun({TestDir,Suite} = TS) -> 1861 case run_make(suites, TestDir, 1862 Suite, UserInclude, 1863 [nowarn_export_all]) of 1864 {error,{make_failed,Bad}} -> 1865 [{TS,Bad}]; 1866 {error,_} -> 1867 [{TS,[filename:join(TestDir, 1868 "*_SUITE")]}]; 1869 _ -> 1870 [] 1871 end 1872 end, TestSuites), 1873 1874 %% try to compile other modules than SUITEs in the test directories 1875 {_,HelpMakeErrors} = 1876 lists:foldl( 1877 fun({Dir,Suite}, {Done,Failed}) -> 1878 case lists:member(Dir, Done) of 1879 false -> 1880 Failed1 = 1881 case run_make(helpmods, Dir, Suite, UserInclude, []) of 1882 {error,{make_failed,BadMods}} -> 1883 [{{Dir,all},BadMods}|Failed]; 1884 {error,_} -> 1885 [{{Dir,all},[Dir]}|Failed]; 1886 _ -> 1887 Failed 1888 end, 1889 {[Dir|Done],Failed1}; 1890 true -> % already visited 1891 {Done,Failed} 1892 end 1893 end, {[],[]}, TestSuites), 1894 {SuiteMakeErrors,lists:reverse(HelpMakeErrors)}. 1895 1896%% verify that specified test suites exist (if auto compile is disabled) 1897verify_suites(TestSuites) -> 1898 io:nl(), 1899 Verify = 1900 fun({Dir,Suite}=DS,{Found,NotFound}) -> 1901 case locate_test_dir(Dir, Suite) of 1902 {ok,TestDir} -> 1903 if Suite == all -> 1904 {[DS|Found],NotFound}; 1905 true -> 1906 Beam = filename:join(TestDir, 1907 atom_to_list(Suite)++ 1908 ".beam"), 1909 case filelib:is_regular(Beam) of 1910 true -> 1911 {[DS|Found],NotFound}; 1912 false -> 1913 case code:is_loaded(Suite) of 1914 {file,SuiteFile} -> 1915 %% test suite is already 1916 %% loaded and since 1917 %% auto_compile == false, 1918 %% let's assume the user has 1919 %% loaded the beam file 1920 %% explicitly 1921 ActualDir = 1922 filename:dirname(SuiteFile), 1923 {[{ActualDir,Suite}|Found], 1924 NotFound}; 1925 false -> 1926 Name = 1927 filename:join(TestDir, 1928 atom_to_list( 1929 Suite)), 1930 io:format(user, 1931 "Suite ~w not found " 1932 "in directory ~ts~n", 1933 [Suite,TestDir]), 1934 {Found,[{DS,[Name]}|NotFound]} 1935 end 1936 end 1937 end; 1938 {error,_Reason} -> 1939 case code:is_loaded(Suite) of 1940 {file,SuiteFile} -> 1941 %% test suite is already loaded and since 1942 %% auto_compile == false, let's assume the 1943 %% user has loaded the beam file explicitly 1944 ActualDir = filename:dirname(SuiteFile), 1945 {[{ActualDir,Suite}|Found],NotFound}; 1946 false -> 1947 io:format(user, "Directory ~ts is " 1948 "invalid~n", [Dir]), 1949 Name = filename:join(Dir, atom_to_list(Suite)), 1950 {Found,[{DS,[Name]}|NotFound]} 1951 end 1952 end 1953 end, 1954 {ActualFound,Missing} = lists:foldl(Verify, {[],[]}, TestSuites), 1955 {lists:reverse(ActualFound),lists:reverse(Missing)}. 1956 1957save_make_errors([]) -> 1958 []; 1959save_make_errors(Errors) -> 1960 Suites = get_bad_suites(Errors,[]), 1961 ct_logs:log("MAKE RESULTS", 1962 "Error compiling or locating the " 1963 "following suites: ~n~p",[Suites]), 1964 %% save the info for logger 1965 ok = file:write_file(?missing_suites_info,term_to_binary(Errors)), 1966 Errors. 1967 1968get_bad_suites([{{_TestDir,_Suite},Failed}|Errors], BadSuites) -> 1969 get_bad_suites(Errors,BadSuites++Failed); 1970get_bad_suites([], BadSuites) -> 1971 BadSuites. 1972 1973 1974step(TestDir, Suite, Case) -> 1975 step(TestDir, Suite, Case, []). 1976 1977step(TestDir, Suite, Case, Opts) when is_list(TestDir), 1978 is_atom(Suite), is_atom(Case), 1979 Suite =/= all, Case =/= all -> 1980 do_run([{TestDir,Suite,Case}], [{step,Opts}]). 1981 1982 1983%%%----------------------------------------------------------------- 1984%%% Internal 1985suite_tuples([{TestDir,Suites,_} | Tests]) when is_list(Suites) -> 1986 lists:map(fun(S) -> {TestDir,S} end, Suites) ++ suite_tuples(Tests); 1987suite_tuples([{TestDir,Suite,_} | Tests]) when is_atom(Suite) -> 1988 [{TestDir,Suite} | suite_tuples(Tests)]; 1989suite_tuples([]) -> 1990 []. 1991 1992final_tests(Tests, Skip, Bad) -> 1993 {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad), 1994 Skip2 = final_skip(Skip1, []), 1995 {Tests1,Skip2}. 1996 1997final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when 1998 is_list(Suites), is_atom(hd(Suites)) -> 1999 Skip1 = [{TD,S,make_failed} || {{TD,S},_} <- Bad, S1 <- Suites, 2000 S == S1, TD == TestDir], 2001 Final1 = [{TestDir,S,all} || S <- Suites], 2002 final_tests1(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad); 2003 2004final_tests1([{TestDir,all,all}|Tests], Final, Skip, Bad) -> 2005 MissingSuites = 2006 case lists:keysearch({TestDir,all}, 1, Bad) of 2007 {value,{_,Failed}} -> 2008 [list_to_atom(filename:basename(F)) || F <- Failed]; 2009 false -> 2010 [] 2011 end, 2012 Missing = [{TestDir,S,make_failed} || S <- MissingSuites], 2013 Final1 = [{TestDir,all,all}|Final], 2014 final_tests1(Tests, Final1, Skip++Missing, Bad); 2015 2016final_tests1([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) when 2017 Cases==[]; Cases==all -> 2018 final_tests1([{TestDir,[Suite],all}|Tests], Final, Skip, Bad); 2019 2020final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when 2021 is_list(GrsOrCs) -> 2022 case lists:keymember({TestDir,Suite}, 1, Bad) of 2023 true -> 2024 Skip1 = Skip ++ [{TestDir,Suite,all,make_failed}], 2025 final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad); 2026 false -> 2027 GrsOrCs1 = 2028 lists:flatmap( 2029 %% for now, only flat group defs are allowed as 2030 %% start options and test spec terms 2031 fun({all,all}) -> 2032 [ct_groups:make_conf(TestDir, Suite, all, [], all)]; 2033 ({skipped,Group,TCs}) -> 2034 [ct_groups:make_conf(TestDir, Suite, 2035 Group, [skipped], TCs)]; 2036 ({skipped,TC}) -> 2037 case lists:member(TC, GrsOrCs) of 2038 true -> 2039 []; 2040 false -> 2041 [TC] 2042 end; 2043 ({GrSpec = {GroupName,_},TCs}) -> 2044 Props = [{override,GrSpec}], 2045 [ct_groups:make_conf(TestDir, Suite, 2046 GroupName, Props, TCs)]; 2047 ({GrSpec = {GroupName,_,_},TCs}) -> 2048 Props = [{override,GrSpec}], 2049 [ct_groups:make_conf(TestDir, Suite, 2050 GroupName, Props, TCs)]; 2051 ({GroupOrGroups,TCs}) -> 2052 [ct_groups:make_conf(TestDir, Suite, 2053 GroupOrGroups, [], TCs)]; 2054 (TC) -> 2055 [TC] 2056 end, GrsOrCs), 2057 Do = {TestDir,Suite,GrsOrCs1}, 2058 final_tests1(Tests, [Do|Final], Skip, Bad) 2059 end; 2060 2061final_tests1([], Final, Skip, _Bad) -> 2062 {lists:reverse(Final),Skip}. 2063 2064final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) -> 2065 SkipConf = ct_groups:make_conf(TestDir, Suite, all, [], all), 2066 Skip = {TestDir,Suite,SkipConf,Reason}, 2067 final_skip(Skips, [Skip|Final]); 2068 2069final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> 2070 Conf = ct_groups:make_conf(TestDir, Suite, Group, [], TCs), 2071 Skip = {TestDir,Suite,Conf,Reason}, 2072 final_skip(Skips, [Skip|Final]); 2073 2074final_skip([Skip|Skips], Final) -> 2075 final_skip(Skips, [Skip|Final]); 2076 2077final_skip([], Final) -> 2078 lists:reverse(Final). 2079 2080continue([], _) -> 2081 true; 2082continue(_MakeErrors, true) -> 2083 false; 2084continue(_MakeErrors, _AbortIfMissingSuites) -> 2085 io:nl(), 2086 OldGL = group_leader(), 2087 case set_group_leader_same_as_shell(OldGL) of 2088 true -> 2089 S = self(), 2090 io:format("Failed to compile or locate one " 2091 "or more test suites\n" 2092 "Press \'c\' to continue or \'a\' to abort.\n" 2093 "Will continue in 15 seconds if no " 2094 "answer is given!\n"), 2095 Pid = spawn(fun() -> 2096 case io:get_line('(c/a) ') of 2097 "c\n" -> 2098 S ! true; 2099 _ -> 2100 S ! false 2101 end 2102 end), 2103 group_leader(OldGL, self()), 2104 receive R when R==true; R==false -> 2105 R 2106 after 15000 -> 2107 exit(Pid, kill), 2108 io:format("... timeout - continuing!!\n"), 2109 true 2110 end; 2111 false -> % no shell process to use 2112 true 2113 end. 2114 2115set_group_leader_same_as_shell(OldGL) -> 2116 %% find the group leader process on the node in a dirty fashion 2117 %% (check initial function call and look in the process dictionary) 2118 GS2or3 = fun(P) -> 2119 case process_info(P,initial_call) of 2120 {initial_call,{group,server,X}} when X == 2 ; X == 3 -> 2121 true; 2122 _ -> 2123 false 2124 end 2125 end, 2126 case [P || P <- processes(), GS2or3(P), 2127 true == lists:keymember(shell,1, 2128 element(2,process_info(P,dictionary)))] of 2129 [GL|_] -> 2130 %% check if started from remote node (skip interaction) 2131 if node(OldGL) /= node(GL) -> false; 2132 true -> group_leader(GL, self()) 2133 end; 2134 [] -> 2135 false 2136 end. 2137 2138check_and_add([{TestDir0,M,_} | Tests], Added, PA) -> 2139 case locate_test_dir(TestDir0, M) of 2140 {ok,TestDir} -> 2141 case lists:member(TestDir, Added) of 2142 true -> 2143 check_and_add(Tests, Added, PA); 2144 false -> 2145 case lists:member(rm_trailing_slash(TestDir), 2146 code:get_path()) of 2147 false -> 2148 true = code:add_patha(TestDir), 2149 check_and_add(Tests, [TestDir|Added], [TestDir|PA]); 2150 true -> 2151 check_and_add(Tests, [TestDir|Added], PA) 2152 end 2153 end; 2154 {error,_} -> 2155 {error,{invalid_directory,TestDir0}} 2156 end; 2157check_and_add([], _, PA) -> 2158 {ok,PA}. 2159 2160do_run_test(Tests, Skip, Opts0) -> 2161 case check_and_add(Tests, [], []) of 2162 {ok,AddedToPath} -> 2163 ct_util:set_testdata({stats,{0,0,{0,0}}}), 2164 2165 %% test_server needs to know the include path too 2166 InclPath = case application:get_env(common_test, include) of 2167 {ok,Incls} -> Incls; 2168 _ -> [] 2169 end, 2170 application:set_env(test_server, include, InclPath), 2171 2172 %% copy the escape characters setting to test_server 2173 EscChars = 2174 case application:get_env(common_test, esc_chars) of 2175 {ok,ECBool} -> ECBool; 2176 _ -> true 2177 end, 2178 application:set_env(test_server, esc_chars, EscChars), 2179 2180 {ok, _} = test_server_ctrl:start_link(local), 2181 2182 %% let test_server expand the test tuples and count no of cases 2183 {Suites,NoOfCases} = count_test_cases(Tests, Skip), 2184 Suites1 = delete_dups(Suites), 2185 NoOfTests = length(Tests), 2186 NoOfSuites = length(Suites1), 2187 ct_util:warn_duplicates(Suites1), 2188 {ok,Cwd} = file:get_cwd(), 2189 io:format("~nCWD set to: ~tp~n", [Cwd]), 2190 if NoOfCases == unknown -> 2191 io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n", 2192 [NoOfTests,NoOfSuites]), 2193 ct_logs:log("TEST INFO","~w test(s), ~w suite(s)", 2194 [NoOfTests,NoOfSuites]); 2195 true -> 2196 io:format("~nTEST INFO: ~w test(s), ~w case(s) " 2197 "in ~w suite(s)~n~n", 2198 [NoOfTests,NoOfCases,NoOfSuites]), 2199 ct_logs:log("TEST INFO","~w test(s), ~w case(s) " 2200 "in ~w suite(s)", 2201 [NoOfTests,NoOfCases,NoOfSuites]) 2202 end, 2203 %% if the verbosity level is set lower than ?STD_IMPORTANCE, tell 2204 %% test_server to ignore stdout printouts to the test case log file 2205 case proplists:get_value(default, Opts0#opts.verbosity) of 2206 VLvl when is_integer(VLvl), (?STD_IMPORTANCE < (100-VLvl)) -> 2207 test_server_ctrl:reject_io_reqs(true); 2208 _Lower -> 2209 ok 2210 end, 2211 2212 case Opts0#opts.multiply_timetraps of 2213 undefined -> MultTT = 1; 2214 MultTT -> MultTT 2215 end, 2216 case Opts0#opts.scale_timetraps of 2217 undefined -> ScaleTT = false; 2218 ScaleTT -> ScaleTT 2219 end, 2220 ct_logs:log("TEST INFO","Timetrap time multiplier = ~w~n" 2221 "Timetrap scaling enabled = ~w", [MultTT,ScaleTT]), 2222 test_server_ctrl:multiply_timetraps(MultTT), 2223 test_server_ctrl:scale_timetraps(ScaleTT), 2224 2225 test_server_ctrl:create_priv_dir(choose_val( 2226 Opts0#opts.create_priv_dir, 2227 auto_per_run)), 2228 2229 {ok,LogDir} = ct_logs:get_log_dir(true), 2230 {TsCoverInfo,Opts} = maybe_start_cover(Opts0, LogDir), 2231 2232 ct_event:notify(#event{name=start_info, 2233 node=node(), 2234 data={NoOfTests,NoOfSuites,NoOfCases}}), 2235 CleanUp = add_jobs(Tests, Skip, Opts, []), 2236 unlink(whereis(test_server_ctrl)), 2237 catch test_server_ctrl:wait_finish(), 2238 2239 maybe_stop_cover(Opts, TsCoverInfo, LogDir), 2240 2241 %% check if last testcase has left a "dead" trace window 2242 %% behind, and if so, kill it 2243 case ct_util:get_testdata(interpret) of 2244 {_What,kill,{TCPid,AttPid}} -> 2245 ct_util:kill_attached(TCPid, AttPid); 2246 _ -> 2247 ok 2248 end, 2249 lists:foreach(fun(Suite) -> 2250 maybe_cleanup_interpret(Suite, Opts#opts.step) 2251 end, CleanUp), 2252 _ = [code:del_path(Dir) || Dir <- AddedToPath], 2253 2254 %% If a severe error has occurred in the test_server, 2255 %% we will generate an exception here. 2256 case ct_util:get_testdata(severe_error) of 2257 undefined -> ok; 2258 SevereError -> 2259 ct_logs:log("SEVERE ERROR", "~tp\n", [SevereError]), 2260 exit(SevereError) 2261 end, 2262 2263 case ct_util:get_testdata(stats) of 2264 Stats = {_Ok,_Failed,{_UserSkipped,_AutoSkipped}} -> 2265 Stats; 2266 _ -> 2267 {error,test_result_unknown} 2268 end; 2269 Error -> 2270 exit(Error) 2271 end. 2272 2273maybe_start_cover(Opts=#opts{cover=Cover,cover_stop=CoverStop0},LogDir) -> 2274 if Cover == undefined -> 2275 {undefined,Opts}; 2276 true -> 2277 case ct_cover:get_spec(Cover) of 2278 {error,Reason} -> 2279 exit({error,Reason}); 2280 CoverSpec -> 2281 CoverStop = 2282 case CoverStop0 of 2283 undefined -> true; 2284 Stop -> Stop 2285 end, 2286 start_cover(Opts#opts{coverspec=CoverSpec, 2287 cover_stop=CoverStop}, 2288 LogDir) 2289 end 2290 end. 2291 2292start_cover(Opts=#opts{coverspec=CovData,cover_stop=CovStop},LogDir) -> 2293 {CovFile, 2294 CovNodes, 2295 CovImport, 2296 _CovExport, 2297 #cover{app = CovApp, 2298 local_only = LocalOnly, 2299 level = CovLevel, 2300 excl_mods = CovExcl, 2301 incl_mods = CovIncl, 2302 cross = CovCross, 2303 src = _CovSrc}} = CovData, 2304 case LocalOnly of 2305 true -> cover:local_only(); 2306 false -> ok 2307 end, 2308 ct_logs:log("COVER INFO", 2309 "Using cover specification file: ~ts~n" 2310 "App: ~w~n" 2311 "Local only: ~w~n" 2312 "Cross cover: ~w~n" 2313 "Including ~w modules~n" 2314 "Excluding ~w modules", 2315 [CovFile,CovApp,LocalOnly,CovCross, 2316 length(CovIncl),length(CovExcl)]), 2317 2318 %% Tell test_server to print a link in its coverlog 2319 %% pointing to the real coverlog which will be written in 2320 %% maybe_stop_cover/2 2321 test_server_ctrl:cover({log,LogDir}), 2322 2323 %% Cover compile all modules 2324 {ok,TsCoverInfo} = test_server_ctrl:cover_compile(CovApp,CovFile, 2325 CovExcl,CovIncl, 2326 CovCross,CovLevel, 2327 CovStop), 2328 ct_logs:log("COVER INFO", 2329 "Compilation completed - test_server cover info: ~tp", 2330 [TsCoverInfo]), 2331 2332 %% start cover on specified nodes 2333 if (CovNodes /= []) and (CovNodes /= undefined) -> 2334 ct_logs:log("COVER INFO", 2335 "Nodes included in cover " 2336 "session: ~tw", 2337 [CovNodes]), 2338 cover:start(CovNodes); 2339 true -> 2340 ok 2341 end, 2342 lists:foreach( 2343 fun(Imp) -> 2344 case cover:import(Imp) of 2345 ok -> 2346 ok; 2347 {error,Reason} -> 2348 ct_logs:log("COVER INFO", 2349 "Importing cover data from: ~ts fails! " 2350 "Reason: ~tp", [Imp,Reason]) 2351 end 2352 end, CovImport), 2353 {TsCoverInfo,Opts}. 2354 2355maybe_stop_cover(_,undefined,_) -> 2356 ok; 2357maybe_stop_cover(#opts{coverspec=CovData},TsCoverInfo,LogDir) -> 2358 {_CovFile, 2359 _CovNodes, 2360 _CovImport, 2361 CovExport, 2362 _AppData} = CovData, 2363 case CovExport of 2364 undefined -> ok; 2365 _ -> 2366 ct_logs:log("COVER INFO","Exporting cover data to ~tp",[CovExport]), 2367 cover:export(CovExport) 2368 end, 2369 ct_logs:log("COVER INFO","Analysing cover data to ~tp",[LogDir]), 2370 test_server_ctrl:cover_analyse(TsCoverInfo,LogDir), 2371 ct_logs:log("COVER INFO","Analysis completed.",[]), 2372 ok. 2373 2374 2375delete_dups([S | Suites]) -> 2376 Suites1 = lists:delete(S, Suites), 2377 [S | delete_dups(Suites1)]; 2378delete_dups([]) -> 2379 []. 2380 2381count_test_cases(Tests, Skip) -> 2382 SendResult = fun(Me, Result) -> Me ! {no_of_cases,Result} end, 2383 TSPid = test_server_ctrl:start_get_totals(SendResult), 2384 Ref = erlang:monitor(process, TSPid), 2385 _ = add_jobs(Tests, Skip, #opts{}, []), 2386 Counted = (catch count_test_cases1(length(Tests), 0, [], Ref)), 2387 erlang:demonitor(Ref, [flush]), 2388 case Counted of 2389 {error,{test_server_died}} = Error -> 2390 throw(Error); 2391 {error,Reason} -> 2392 unlink(whereis(test_server_ctrl)), 2393 test_server_ctrl:stop(), 2394 throw({user_error,Reason}); 2395 Result -> 2396 test_server_ctrl:stop_get_totals(), 2397 Result 2398 end. 2399 2400count_test_cases1(0, N, Suites, _) -> 2401 {lists:flatten(Suites), N}; 2402count_test_cases1(Jobs, N, Suites, Ref) -> 2403 receive 2404 {_,{error,_Reason} = Error} -> 2405 throw(Error); 2406 {no_of_cases,{Ss,N1}} -> 2407 count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref); 2408 {'DOWN', Ref, _, _, Info} -> 2409 throw({error,{test_server_died,Info}}) 2410 end. 2411 2412add_known(unknown, _) -> 2413 unknown; 2414add_known(_, unknown) -> 2415 unknown; 2416add_known(N, N1) -> 2417 N+N1. 2418 2419add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) -> 2420 Name = get_name(TestDir), 2421 case catch test_server_ctrl:add_dir_with_skip(Name, TestDir, 2422 skiplist(TestDir,Skip)) of 2423 {'EXIT',_} -> 2424 CleanUp; 2425 _ -> 2426 case wait_for_idle() of 2427 ok -> 2428 add_jobs(Tests, Skip, Opts, CleanUp); 2429 _ -> 2430 CleanUp 2431 end 2432 end; 2433add_jobs([{TestDir,[Suite],all}|Tests], Skip, 2434 Opts, CleanUp) when is_atom(Suite) -> 2435 add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp); 2436add_jobs([{TestDir,Suites,all}|Tests], Skip, 2437 Opts, CleanUp) when is_list(Suites) -> 2438 Name = get_name(TestDir) ++ ".suites", 2439 case catch test_server_ctrl:add_module_with_skip(Name, Suites, 2440 skiplist(TestDir,Skip)) of 2441 {'EXIT',_} -> 2442 CleanUp; 2443 _ -> 2444 case wait_for_idle() of 2445 ok -> 2446 add_jobs(Tests, Skip, Opts, CleanUp); 2447 _ -> 2448 CleanUp 2449 end 2450 end; 2451add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> 2452 case maybe_interpret(Suite, all, Opts) of 2453 ok -> 2454 Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite), 2455 case catch test_server_ctrl:add_module_with_skip(Name, [Suite], 2456 skiplist(TestDir, 2457 Skip)) of 2458 {'EXIT',_} -> 2459 CleanUp; 2460 _ -> 2461 case wait_for_idle() of 2462 ok -> 2463 add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); 2464 _ -> 2465 CleanUp 2466 end 2467 end; 2468 Error -> 2469 Error 2470 end; 2471 2472%% group (= conf case in test_server) 2473add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when 2474 element(1, hd(Confs)) == conf -> 2475 Group = fun(Conf) -> proplists:get_value(name, element(2, Conf)) end, 2476 TestCases = fun(Conf) -> element(4, Conf) end, 2477 TCTestName = fun(all) -> ""; 2478 ([C]) when is_atom(C) -> "." ++ atom_to_list(C); 2479 (Cs) when is_list(Cs) -> ".cases" 2480 end, 2481 GrTestName = 2482 case Confs of 2483 [Conf] -> 2484 case Group(Conf) of 2485 GrName when is_atom(GrName) -> 2486 "." ++ atom_to_list(GrName) ++ 2487 TCTestName(TestCases(Conf)); 2488 _ -> 2489 ".groups" ++ TCTestName(TestCases(Conf)) 2490 end; 2491 _ -> 2492 ".groups" 2493 end, 2494 TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName, 2495 case maybe_interpret(Suite, init_per_group, Opts) of 2496 ok -> 2497 case catch test_server_ctrl:add_conf_with_skip(TestName, 2498 Suite, 2499 Confs, 2500 skiplist(TestDir, 2501 Skip)) of 2502 {'EXIT',_} -> 2503 CleanUp; 2504 _ -> 2505 case wait_for_idle() of 2506 ok -> 2507 add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); 2508 _ -> 2509 CleanUp 2510 end 2511 end; 2512 Error -> 2513 Error 2514 end; 2515 2516%% test case 2517add_jobs([{TestDir,Suite,[Case]}|Tests], 2518 Skip, Opts, CleanUp) when is_atom(Case) -> 2519 add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp); 2520 2521add_jobs([{TestDir,Suite,Cases}|Tests], 2522 Skip, Opts, CleanUp) when is_list(Cases) -> 2523 Cases1 = lists:map(fun({GroupName,_}) when is_atom(GroupName) -> GroupName; 2524 (Case) -> Case 2525 end, Cases), 2526 case maybe_interpret(Suite, Cases1, Opts) of 2527 ok -> 2528 Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases", 2529 case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases1, 2530 skiplist(TestDir, 2531 Skip)) of 2532 {'EXIT',_} -> 2533 CleanUp; 2534 _ -> 2535 case wait_for_idle() of 2536 ok -> 2537 add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); 2538 _ -> 2539 CleanUp 2540 end 2541 end; 2542 Error -> 2543 Error 2544 end; 2545add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -> 2546 case maybe_interpret(Suite, Case, Opts) of 2547 ok -> 2548 Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++ 2549 atom_to_list(Case), 2550 case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case, 2551 skiplist(TestDir, 2552 Skip)) of 2553 {'EXIT',_} -> 2554 CleanUp; 2555 _ -> 2556 case wait_for_idle() of 2557 ok -> 2558 add_jobs(Tests, Skip, Opts, [Suite|CleanUp]); 2559 _ -> 2560 CleanUp 2561 end 2562 end; 2563 Error -> 2564 Error 2565 end; 2566add_jobs([], _, _, CleanUp) -> 2567 CleanUp. 2568 2569wait_for_idle() -> 2570 ct_util:update_last_run_index(), 2571 Notify = fun(Me,IdleState) -> Me ! {idle,IdleState}, 2572 receive 2573 {Me,proceed} -> ok 2574 after 2575 30000 -> ok 2576 end 2577 end, 2578 case catch test_server_ctrl:idle_notify(Notify) of 2579 {'EXIT',_} -> 2580 error; 2581 TSPid -> 2582 %% so we don't hang forever if test_server dies 2583 Ref = erlang:monitor(process, TSPid), 2584 Result = receive 2585 {idle,abort} -> aborted; 2586 {idle,_} -> ok; 2587 {'DOWN', Ref, _, _, _} -> error 2588 end, 2589 erlang:demonitor(Ref, [flush]), 2590 ct_util:update_last_run_index(), 2591 %% let test_server_ctrl proceed (and possibly shut down) now 2592 TSPid ! {self(),proceed}, 2593 Result 2594 end. 2595 2596skiplist(Dir, [{Dir,all,Cmt}|Skip]) -> 2597 %% we need to turn 'all' into list of modules since 2598 %% test_server doesn't do skips on Dir level 2599 Ss = filelib:wildcard(filename:join(Dir, "*_SUITE.beam")), 2600 [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ 2601 skiplist(Dir,Skip); 2602skiplist(Dir, [{Dir,S,Cmt}|Skip]) -> 2603 [{S,Cmt} | skiplist(Dir, Skip)]; 2604skiplist(Dir, [{Dir,S,C,Cmt}|Skip]) -> 2605 [{S,C,Cmt} | skiplist(Dir, Skip)]; 2606skiplist(Dir, [_|Skip]) -> 2607 skiplist(Dir, Skip); 2608skiplist(_Dir, []) -> 2609 []. 2610 2611get_name(Dir) -> 2612 TestDir = 2613 case filename:basename(Dir) of 2614 "test" -> 2615 filename:dirname(Dir); 2616 _ -> 2617 Dir 2618 end, 2619 Base = filename:basename(TestDir), 2620 case filename:basename(filename:dirname(TestDir)) of 2621 "" -> 2622 Base; 2623 TopDir -> 2624 TopDir ++ "." ++ Base 2625 end. 2626 2627run_make(TestDir, Mod, UserInclude) -> 2628 run_make(suites, TestDir, Mod, UserInclude, [nowarn_export_all]). 2629 2630run_make(Targets, TestDir0, Mod, UserInclude, COpts) when is_list(Mod) -> 2631 run_make(Targets, TestDir0, list_to_atom(Mod), UserInclude, COpts); 2632 2633run_make(Targets, TestDir0, Mod, UserInclude, COpts) -> 2634 case locate_test_dir(TestDir0, Mod) of 2635 {ok,TestDir} -> 2636 %% send a start_make notification which may suspend 2637 %% the process if some other node is compiling files 2638 %% in the same directory 2639 ct_event:sync_notify(#event{name=start_make, 2640 node=node(), 2641 data=TestDir}), 2642 {ok,Cwd} = file:get_cwd(), 2643 ok = file:set_cwd(TestDir), 2644 CtInclude = get_dir(common_test, "include"), 2645 XmerlInclude = get_dir(xmerl, "include"), 2646 ErlFlags = UserInclude ++ [{i,CtInclude}, 2647 {i,XmerlInclude}, 2648 debug_info] ++ COpts, 2649 Result = 2650 if Mod == all ; Targets == helpmods -> 2651 case (catch ct_make:all([noexec|ErlFlags])) of 2652 {'EXIT',_} = Failure -> 2653 Failure; 2654 MakeInfo -> 2655 FileTest = fun(F, suites) -> is_suite(F); 2656 (F, helpmods) -> not is_suite(F) 2657 end, 2658 Files = 2659 lists:flatmap(fun({F,out_of_date}) -> 2660 case FileTest(F, 2661 Targets) of 2662 true -> [F]; 2663 false -> [] 2664 end; 2665 (_) -> 2666 [] 2667 end, MakeInfo), 2668 (catch ct_make:files(Files, [load|ErlFlags])) 2669 end; 2670 true -> 2671 (catch ct_make:files([Mod], [load|ErlFlags])) 2672 end, 2673 2674 ok = file:set_cwd(Cwd), 2675 %% send finished_make notification 2676 ct_event:notify(#event{name=finished_make, 2677 node=node(), 2678 data=TestDir}), 2679 case Result of 2680 {up_to_date,_} -> 2681 ok; 2682 {'EXIT',Reason} -> 2683 io:format("{error,{make_crashed,~tp}\n", [Reason]), 2684 {error,{make_crashed,TestDir,Reason}}; 2685 {error,ModInfo} -> 2686 io:format("{error,make_failed}\n", []), 2687 Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo, 2688 R == error], 2689 {error,{make_failed,Bad}} 2690 end; 2691 {error,_} -> 2692 io:format("{error,{invalid_directory,~tp}}\n", [TestDir0]), 2693 {error,{invalid_directory,TestDir0}} 2694 end. 2695 2696get_dir(App, Dir) -> 2697 filename:join(code:lib_dir(App), Dir). 2698 2699maybe_interpret(Suite, Cases, #opts{step = StepOpts}) when StepOpts =/= undefined -> 2700 %% if other suite has run before this one, check if last testcase 2701 %% has left a "dead" trace window behind, and if so, kill it 2702 case ct_util:get_testdata(interpret) of 2703 {_What,kill,{TCPid,AttPid}} -> 2704 ct_util:kill_attached(TCPid, AttPid); 2705 _ -> 2706 ok 2707 end, 2708 maybe_interpret1(Suite, Cases, StepOpts); 2709maybe_interpret(_, _, _) -> 2710 ok. 2711 2712maybe_interpret1(Suite, all, StepOpts) -> 2713 case i:ii(Suite) of 2714 {module,_} -> 2715 i:iaa([break]), 2716 case get_all_testcases(Suite) of 2717 {error,_} -> 2718 {error,no_testcases_found}; 2719 Cases -> 2720 maybe_interpret2(Suite, Cases, StepOpts) 2721 end; 2722 error -> 2723 {error,could_not_interpret_module} 2724 end; 2725maybe_interpret1(Suite, Case, StepOpts) when is_atom(Case) -> 2726 maybe_interpret1(Suite, [Case], StepOpts); 2727maybe_interpret1(Suite, Cases, StepOpts) when is_list(Cases) -> 2728 case i:ii(Suite) of 2729 {module,_} -> 2730 i:iaa([break]), 2731 maybe_interpret2(Suite, Cases, StepOpts); 2732 error -> 2733 {error,could_not_interpret_module} 2734 end. 2735 2736maybe_interpret2(Suite, Cases, StepOpts) -> 2737 set_break_on_config(Suite, StepOpts), 2738 _ = [begin try i:ib(Suite, Case, 1) of 2739 _ -> ok 2740 catch 2741 _:_Error -> 2742 io:format(user, "Invalid breakpoint: ~w:~tw/1~n", 2743 [Suite,Case]) 2744 end 2745 end || Case <- Cases, is_atom(Case)], 2746 test_server_ctrl:multiply_timetraps(infinity), 2747 WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of 2748 true -> no_kill; 2749 false -> kill 2750 end, 2751 ct_util:set_testdata({interpret,{{Suite,Cases},WinOp, 2752 {undefined,undefined}}}), 2753 ok. 2754 2755set_break_on_config(Suite, StepOpts) -> 2756 case lists:member(config, ensure_atom(StepOpts)) of 2757 true -> 2758 SetBPIfExists = fun(F,A) -> 2759 case erlang:function_exported(Suite, F, A) of 2760 true -> i:ib(Suite, F, A); 2761 false -> ok 2762 end 2763 end, 2764 ok = SetBPIfExists(init_per_suite, 1), 2765 ok = SetBPIfExists(init_per_group, 2), 2766 ok = SetBPIfExists(init_per_testcase, 2), 2767 ok = SetBPIfExists(end_per_testcase, 2), 2768 ok = SetBPIfExists(end_per_group, 2), 2769 ok = SetBPIfExists(end_per_suite, 1); 2770 false -> 2771 ok 2772 end. 2773 2774maybe_cleanup_interpret(_, undefined) -> 2775 ok; 2776maybe_cleanup_interpret(Suite, _) -> 2777 i:iq(Suite). 2778 2779log_ts_names([]) -> 2780 ok; 2781log_ts_names(Specs) -> 2782 List = lists:map(fun(Name) -> 2783 Name ++ " " 2784 end, Specs), 2785 ct_logs:log("Test Specification file(s)", "~ts", 2786 [lists:flatten(List)]). 2787 2788merge_arguments(Args) -> 2789 merge_arguments(Args, []). 2790 2791merge_arguments([LogDir={logdir,_}|Args], Merged) -> 2792 merge_arguments(Args, handle_arg(replace, LogDir, Merged)); 2793 2794merge_arguments([CoverFile={cover,_}|Args], Merged) -> 2795 merge_arguments(Args, handle_arg(replace, CoverFile, Merged)); 2796 2797merge_arguments([CoverStop={cover_stop,_}|Args], Merged) -> 2798 merge_arguments(Args, handle_arg(replace, CoverStop, Merged)); 2799 2800merge_arguments([{'case',TC}|Args], Merged) -> 2801 merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged)); 2802 2803merge_arguments([Arg|Args], Merged) -> 2804 merge_arguments(Args, handle_arg(merge, Arg, Merged)); 2805 2806merge_arguments([], Merged) -> 2807 Merged. 2808 2809handle_arg(replace, {Key,Elems}, [{Key,_}|Merged]) -> 2810 [{Key,Elems}|Merged]; 2811handle_arg(merge, {event_handler_init,Elems}, [{event_handler_init,PrevElems}|Merged]) -> 2812 [{event_handler_init,PrevElems++["add"|Elems]}|Merged]; 2813handle_arg(merge, {userconfig,Elems}, [{userconfig,PrevElems}|Merged]) -> 2814 [{userconfig,PrevElems++["add"|Elems]}|Merged]; 2815handle_arg(merge, {Key,Elems}, [{Key,PrevElems}|Merged]) -> 2816 [{Key,PrevElems++Elems}|Merged]; 2817handle_arg(Op, Arg, [Other|Merged]) -> 2818 [Other|handle_arg(Op, Arg, Merged)]; 2819handle_arg(_,Arg,[]) -> 2820 [Arg]. 2821 2822get_start_opt(Key, IfExists, Args) -> 2823 get_start_opt(Key, IfExists, undefined, Args). 2824 2825get_start_opt(Key, IfExists, IfNotExists, Args) -> 2826 try try_get_start_opt(Key, IfExists, IfNotExists, Args) of 2827 Result -> 2828 Result 2829 catch 2830 error:_ -> 2831 exit({user_error,{bad_argument,Key}}) 2832 end. 2833 2834try_get_start_opt(Key, IfExists, IfNotExists, Args) -> 2835 case lists:keysearch(Key, 1, Args) of 2836 {value,{Key,Val}} when is_function(IfExists) -> 2837 IfExists(Val); 2838 {value,{Key,Val}} when IfExists == value -> 2839 Val; 2840 {value,{Key,_Val}} -> 2841 IfExists; 2842 _ -> 2843 IfNotExists 2844 end. 2845 2846ct_hooks_args2opts(Args) -> 2847 lists:foldl(fun({ct_hooks,Hooks}, Acc) -> 2848 ct_hooks_args2opts(Hooks,Acc); 2849 (_,Acc) -> 2850 Acc 2851 end,[],Args). 2852 2853ct_hooks_args2opts([CTH,Arg,Prio,"and"| Rest],Acc) when Arg /= "and" -> 2854 ct_hooks_args2opts(Rest,[{list_to_atom(CTH), 2855 parse_cth_args(Arg), 2856 parse_cth_args(Prio)}|Acc]); 2857ct_hooks_args2opts([CTH,Arg,"and"| Rest],Acc) -> 2858 ct_hooks_args2opts(Rest,[{list_to_atom(CTH), 2859 parse_cth_args(Arg)}|Acc]); 2860ct_hooks_args2opts([CTH], Acc) -> 2861 ct_hooks_args2opts([CTH,"and"],Acc); 2862ct_hooks_args2opts([CTH, "and" | Rest], Acc) -> 2863 ct_hooks_args2opts(Rest,[list_to_atom(CTH)|Acc]); 2864ct_hooks_args2opts([CTH, Args], Acc) -> 2865 ct_hooks_args2opts([CTH, Args, "and"],Acc); 2866ct_hooks_args2opts([CTH, Args, Prio], Acc) -> 2867 ct_hooks_args2opts([CTH, Args, Prio, "and"],Acc); 2868ct_hooks_args2opts([],Acc) -> 2869 lists:reverse(Acc). 2870 2871parse_cth_args(String) -> 2872 try 2873 true = io_lib:printable_unicode_list(String), 2874 {ok,Toks,_} = erl_scan:string(String++"."), 2875 {ok, Args} = erl_parse:parse_term(Toks), 2876 Args 2877 catch _:_ -> 2878 String 2879 end. 2880 2881event_handler_args2opts(Args) -> 2882 case proplists:get_value(event_handler, Args) of 2883 undefined -> 2884 event_handler_args2opts([], Args); 2885 EHs -> 2886 event_handler_args2opts([{list_to_atom(EH),[]} || EH <- EHs], Args) 2887 end. 2888event_handler_args2opts(Default, Args) -> 2889 case proplists:get_value(event_handler_init, Args) of 2890 undefined -> 2891 Default; 2892 EHs -> 2893 event_handler_init_args2opts(EHs) 2894 end. 2895event_handler_init_args2opts([EH, Arg, "and" | EHs]) -> 2896 [{list_to_atom(EH),lists:flatten(io_lib:format("~ts",[Arg]))} | 2897 event_handler_init_args2opts(EHs)]; 2898event_handler_init_args2opts([EH, Arg]) -> 2899 [{list_to_atom(EH),lists:flatten(io_lib:format("~ts",[Arg]))}]; 2900event_handler_init_args2opts([]) -> 2901 []. 2902 2903verbosity_args2opts(Args) -> 2904 case proplists:get_value(verbosity, Args) of 2905 undefined -> 2906 []; 2907 VArgs -> 2908 GetVLvls = 2909 fun("and", {new,SoFar}) when is_list(SoFar) -> 2910 {new,SoFar}; 2911 ("and", {Lvl,SoFar}) when is_list(SoFar) -> 2912 {new,[{'$unspecified',list_to_integer(Lvl)} | SoFar]}; 2913 (CatOrLvl, {new,SoFar}) when is_list(SoFar) -> 2914 {CatOrLvl,SoFar}; 2915 (Lvl, {Cat,SoFar}) -> 2916 {new,[{list_to_atom(Cat),list_to_integer(Lvl)} | SoFar]} 2917 end, 2918 case lists:foldl(GetVLvls, {new,[]}, VArgs) of 2919 {new,Parsed} -> 2920 Parsed; 2921 {Lvl,Parsed} -> 2922 [{'$unspecified',list_to_integer(Lvl)} | Parsed] 2923 end 2924 end. 2925 2926add_verbosity_defaults(VLvls) -> 2927 case {proplists:get_value('$unspecified', VLvls), 2928 proplists:get_value(default, VLvls)} of 2929 {undefined,undefined} -> 2930 ?default_verbosity ++ VLvls; 2931 {Lvl,undefined} -> 2932 [{default,Lvl} | VLvls]; 2933 {undefined,_Lvl} -> 2934 [{'$unspecified',?MAX_VERBOSITY} | VLvls]; 2935 _ -> 2936 VLvls 2937 end. 2938 2939%% This function reads pa and pz arguments, converts dirs from relative 2940%% to absolute, and re-inserts them in the code path. The order of the 2941%% dirs in the code path remain the same. Note however that since this 2942%% function is only used for arguments "pre run_test erl_args", the order 2943%% relative dirs "post run_test erl_args" is not kept! 2944rel_to_abs(CtArgs) -> 2945 {PA,PZ} = get_pa_pz(CtArgs, [], []), 2946 _ = [begin 2947 Dir = rm_trailing_slash(D), 2948 Abs = make_abs(Dir), 2949 _ = if Dir /= Abs -> 2950 _ = code:del_path(Dir), 2951 _ = code:del_path(Abs), 2952 io:format(user, "Converting ~tp to ~tp and re-inserting " 2953 "with add_pathz/1~n", 2954 [Dir, Abs]); 2955 true -> 2956 _ = code:del_path(Dir) 2957 end, 2958 code:add_pathz(Abs) 2959 end || D <- PZ], 2960 _ = [begin 2961 Dir = rm_trailing_slash(D), 2962 Abs = make_abs(Dir), 2963 _ = if Dir /= Abs -> 2964 _ = code:del_path(Dir), 2965 _ = code:del_path(Abs), 2966 io:format(user, "Converting ~tp to ~tp and re-inserting " 2967 "with add_patha/1~n", 2968 [Dir, Abs]); 2969 true -> 2970 _ = code:del_path(Dir) 2971 end, 2972 code:add_patha(Abs) 2973 end || D <- PA], 2974 io:format(user, "~n", []). 2975 2976rm_trailing_slash(Dir) -> 2977 filename:join(filename:split(Dir)). 2978 2979get_pa_pz([{pa,Dirs} | Args], PA, PZ) -> 2980 get_pa_pz(Args, PA ++ Dirs, PZ); 2981get_pa_pz([{pz,Dirs} | Args], PA, PZ) -> 2982 get_pa_pz(Args, PA, PZ ++ Dirs); 2983get_pa_pz([_ | Args], PA, PZ) -> 2984 get_pa_pz(Args, PA, PZ); 2985get_pa_pz([], PA, PZ) -> 2986 {PA,PZ}. 2987 2988make_abs(RelDir) -> 2989 Tokens = filename:split(filename:absname(RelDir)), 2990 filename:join(lists:reverse(make_abs1(Tokens, []))). 2991 2992make_abs1([".."|Dirs], [_Dir|Path]) -> 2993 make_abs1(Dirs, Path); 2994make_abs1(["."|Dirs], Path) -> 2995 make_abs1(Dirs, Path); 2996make_abs1([Dir|Dirs], Path) -> 2997 make_abs1(Dirs, [Dir|Path]); 2998make_abs1([], Path) -> 2999 Path. 3000 3001%% This function translates ct:run_test/1 start options 3002%% to ct_run start arguments (on the init arguments format) - 3003%% this is useful mainly for testing the ct_run start functions. 3004opts2args(EnvStartOpts) -> 3005 lists:flatmap(fun({exit_status,ExitStatusOpt}) when is_atom(ExitStatusOpt) -> 3006 [{exit_status,[atom_to_list(ExitStatusOpt)]}]; 3007 ({halt_with,{HaltM,HaltF}}) -> 3008 [{halt_with,[atom_to_list(HaltM), 3009 atom_to_list(HaltF)]}]; 3010 ({interactive_mode,true}) -> 3011 [{shell,[]}]; 3012 ({config,CfgFile}) when is_integer(hd(CfgFile)) -> 3013 [{ct_config,[CfgFile]}]; 3014 ({config,CfgFiles}) when is_list(hd(CfgFiles)) -> 3015 [{ct_config,CfgFiles}]; 3016 ({userconfig,{CBM,CfgStr=[X|_]}}) when is_integer(X) -> 3017 [{userconfig,[atom_to_list(CBM),CfgStr]}]; 3018 ({userconfig,{CBM,CfgStrs}}) when is_list(CfgStrs) -> 3019 [{userconfig,[atom_to_list(CBM) | CfgStrs]}]; 3020 ({userconfig,UserCfg}) when is_list(UserCfg) -> 3021 Strs = 3022 lists:map(fun({CBM,CfgStr=[X|_]}) 3023 when is_integer(X) -> 3024 [atom_to_list(CBM), 3025 CfgStr,"and"]; 3026 ({CBM,CfgStrs}) 3027 when is_list(CfgStrs) -> 3028 [atom_to_list(CBM) | CfgStrs] ++ 3029 ["and"] 3030 end, UserCfg), 3031 [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), 3032 [{userconfig,lists:reverse(StrsR)}]; 3033 ({group,G}) when is_atom(G) -> 3034 [{group,[atom_to_list(G)]}]; 3035 ({group,Gs}) when is_list(Gs) -> 3036 LOfGStrs = [lists:flatten(io_lib:format("~tw",[G])) || 3037 G <- Gs], 3038 [{group,LOfGStrs}]; 3039 ({testcase,Case}) when is_atom(Case) -> 3040 [{'case',[atom_to_list(Case)]}]; 3041 ({testcase,Cases}) -> 3042 [{'case',[atom_to_list(C) || C <- Cases]}]; 3043 ({'case',Cases}) -> 3044 [{'case',[atom_to_list(C) || C <- Cases]}]; 3045 ({allow_user_terms,true}) -> 3046 [{allow_user_terms,[]}]; 3047 ({allow_user_terms,false}) -> 3048 []; 3049 ({join_specs,true}) -> 3050 [{join_specs,[]}]; 3051 ({join_specs,false}) -> 3052 []; 3053 ({auto_compile,false}) -> 3054 [{no_auto_compile,[]}]; 3055 ({auto_compile,true}) -> 3056 []; 3057 ({scale_timetraps,true}) -> 3058 [{scale_timetraps,[]}]; 3059 ({scale_timetraps,false}) -> 3060 []; 3061 ({create_priv_dir,auto_per_run}) -> 3062 []; 3063 ({create_priv_dir,PD}) when is_atom(PD) -> 3064 [{create_priv_dir,[atom_to_list(PD)]}]; 3065 ({force_stop,skip_rest}) -> 3066 [{force_stop,["skip_rest"]}]; 3067 ({force_stop,true}) -> 3068 [{force_stop,[]}]; 3069 ({force_stop,false}) -> 3070 []; 3071 ({decrypt,{key,Key}}) -> 3072 [{ct_decrypt_key,[Key]}]; 3073 ({decrypt,{file,File}}) -> 3074 [{ct_decrypt_file,[File]}]; 3075 ({basic_html,true}) -> 3076 [{basic_html,[]}]; 3077 ({basic_html,false}) -> 3078 []; 3079 ({esc_chars,false}) -> 3080 [{no_esc_chars,[]}]; 3081 ({esc_chars,true}) -> 3082 []; 3083 ({event_handler,EH}) when is_atom(EH) -> 3084 [{event_handler,[atom_to_list(EH)]}]; 3085 ({event_handler,EHs}) when is_list(EHs) -> 3086 [{event_handler,[atom_to_list(EH) || EH <- EHs]}]; 3087 ({event_handler,{EH,Arg}}) when is_atom(EH) -> 3088 ArgStr = lists:flatten(io_lib:format("~tp", [Arg])), 3089 [{event_handler_init,[atom_to_list(EH),ArgStr]}]; 3090 ({event_handler,{EHs,Arg}}) when is_list(EHs) -> 3091 ArgStr = lists:flatten(io_lib:format("~tp", [Arg])), 3092 Strs = lists:flatmap(fun(EH) -> 3093 [atom_to_list(EH), 3094 ArgStr,"and"] 3095 end, EHs), 3096 [_LastAnd | StrsR] = lists:reverse(Strs), 3097 [{event_handler_init,lists:reverse(StrsR)}]; 3098 ({logopts,LOs}) when is_list(LOs) -> 3099 [{logopts,[atom_to_list(LO) || LO <- LOs]}]; 3100 ({verbosity,?default_verbosity}) -> 3101 []; 3102 ({verbosity,VLvl}) when is_integer(VLvl) -> 3103 [{verbosity,[integer_to_list(VLvl)]}]; 3104 ({verbosity,VLvls}) when is_list(VLvls) -> 3105 VLvlArgs = 3106 lists:flatmap(fun({'$unspecified',Lvl}) -> 3107 [integer_to_list(Lvl), 3108 "and"]; 3109 ({Cat,Lvl}) -> 3110 [atom_to_list(Cat), 3111 integer_to_list(Lvl), 3112 "and"]; 3113 (Lvl) -> 3114 [integer_to_list(Lvl), 3115 "and"] 3116 end, VLvls), 3117 [_LastAnd|VLvlArgsR] = lists:reverse(VLvlArgs), 3118 [{verbosity,lists:reverse(VLvlArgsR)}]; 3119 ({ct_hooks,[]}) -> 3120 []; 3121 ({ct_hooks,CTHs}) when is_list(CTHs) -> 3122 io:format(user,"ct_hooks: ~tp",[CTHs]), 3123 Strs = lists:flatmap( 3124 fun({CTH,Arg,Prio}) -> 3125 [atom_to_list(CTH), 3126 lists:flatten( 3127 io_lib:format("~tp",[Arg])), 3128 lists:flatten( 3129 io_lib:format("~tp",[Prio])), 3130 "and"]; 3131 ({CTH,Arg}) -> 3132 [atom_to_list(CTH), 3133 lists:flatten( 3134 io_lib:format("~tp",[Arg])), 3135 "and"]; 3136 (CTH) when is_atom(CTH) -> 3137 [atom_to_list(CTH),"and"] 3138 end,CTHs), 3139 [_LastAnd|StrsR] = lists:reverse(Strs), 3140 io:format(user,"return: ~tp",[lists:reverse(StrsR)]), 3141 [{ct_hooks,lists:reverse(StrsR)}]; 3142 ({Opt,As=[A|_]}) when is_atom(A) -> 3143 [{Opt,[atom_to_list(Atom) || Atom <- As]}]; 3144 ({Opt,Strs=[S|_]}) when is_list(S) -> 3145 [{Opt,Strs}]; 3146 ({Opt,A}) when is_atom(A) -> 3147 [{Opt,[atom_to_list(A)]}]; 3148 ({Opt,I}) when is_integer(I) -> 3149 [{Opt,[integer_to_list(I)]}]; 3150 ({Opt,S}) when is_list(S) -> 3151 [{Opt,[S]}]; 3152 (Opt) -> 3153 Opt 3154 end, EnvStartOpts). 3155 3156locate_test_dir(Dir, Suite) -> 3157 TestDir = case ct_util:is_test_dir(Dir) of 3158 true -> Dir; 3159 false -> ct_util:get_testdir(Dir, Suite) 3160 end, 3161 case filelib:is_dir(TestDir) of 3162 true -> {ok,TestDir}; 3163 false -> {error,invalid} 3164 end. 3165 3166is_suite(Mod) when is_atom(Mod) -> 3167 is_suite(atom_to_list(Mod)); 3168is_suite(ModOrFile) when is_list(ModOrFile) -> 3169 case lists:reverse(filename:basename(ModOrFile, ".erl")) of 3170 [$E,$T,$I,$U,$S,$_|_] -> 3171 true; 3172 _ -> 3173 case lists:reverse(filename:basename(ModOrFile, ".beam")) of 3174 [$E,$T,$I,$U,$S,$_|_] -> 3175 true; 3176 _ -> 3177 false 3178 end 3179 end. 3180 3181get_all_testcases(Suite) -> 3182 try ct_framework:get_all_cases(Suite) of 3183 {error,_Reason} = Error -> 3184 Error; 3185 SuiteCases -> 3186 Cases = [C || {_S,C} <- SuiteCases], 3187 try Suite:sequences() of 3188 [] -> 3189 Cases; 3190 Seqs -> 3191 TCs1 = lists:flatten([TCs || {_,TCs} <- Seqs]), 3192 lists:reverse( 3193 lists:foldl(fun(TC, Acc) -> 3194 case lists:member(TC, Acc) of 3195 true -> Acc; 3196 false -> [TC | Acc] 3197 end 3198 end, [], Cases ++ TCs1)) 3199 catch 3200 _:_ -> 3201 Cases 3202 end 3203 catch 3204 _:Error -> 3205 {error,Error} 3206 end. 3207 3208%% Internal tracing support. If {ct_trace,TraceSpec} is present, the 3209%% TraceSpec file will be consulted and dbg used to trace function 3210%% calls during test run. Expected terms in TraceSpec: 3211%% {m,Mod} or {f,Mod,Func}. 3212start_trace(Args) -> 3213 case lists:keysearch(ct_trace,1,Args) of 3214 {value,{ct_trace,File}} -> 3215 TraceSpec = delistify(File), 3216 case file:consult(TraceSpec) of 3217 {ok,Terms} -> 3218 case catch do_trace(Terms) of 3219 ok -> 3220 true; 3221 {_,Error} -> 3222 io:format("Warning! Tracing not started. Reason: ~tp~n~n", 3223 [Error]), 3224 false 3225 end; 3226 {_,Error} -> 3227 io:format("Warning! Tracing not started. Reason: ~ts~n~n", 3228 [file:format_error(Error)]), 3229 false 3230 end; 3231 false -> 3232 false 3233 end. 3234 3235do_trace(Terms) -> 3236 dbg:tracer(), 3237 dbg:p(self(), [sos,call]), 3238 lists:foreach(fun({m,M}) -> 3239 case dbg:tpl(M,x) of 3240 {error,What} -> exit({error,{tracing_failed,What}}); 3241 _ -> ok 3242 end; 3243 ({me,M}) -> 3244 case dbg:tp(M,[{'_',[],[{exception_trace}, 3245 {message,{caller}}]}]) of 3246 {error,What} -> exit({error,{tracing_failed,What}}); 3247 _ -> ok 3248 end; 3249 ({f,M,F}) -> 3250 case dbg:tpl(M,F,[{'_',[],[{exception_trace}, 3251 {message,{caller}}]}]) of 3252 {error,What} -> exit({error,{tracing_failed,What}}); 3253 _ -> ok 3254 end; 3255 (Huh) -> 3256 exit({error,{unrecognized_trace_term,Huh}}) 3257 end, Terms), 3258 ok. 3259 3260stop_trace(true) -> 3261 dbg:stop_clear(); 3262stop_trace(false) -> 3263 ok. 3264 3265ensure_atom(Atom) when is_atom(Atom) -> 3266 Atom; 3267ensure_atom(String) when is_list(String), is_integer(hd(String)) -> 3268 list_to_atom(String); 3269ensure_atom(List) when is_list(List) -> 3270 [ensure_atom(Item) || Item <- List]; 3271ensure_atom(Other) -> 3272 Other. 3273