1%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- 2%% ex: ts=4 sw=4 et 3 4-module(rebar_prv_common_test). 5 6-behaviour(provider). 7 8-export([init/1, 9 do/1, 10 format_error/1]). 11 12-ifdef(TEST). 13%% exported for test purposes 14-export([compile/2, prepare_tests/1, translate_paths/2, maybe_write_coverdata/1]). 15-endif. 16 17-include("rebar.hrl"). 18-include_lib("providers/include/providers.hrl"). 19 20-define(PROVIDER, ct). 21%% we need to modify app_info state before compile 22-define(DEPS, [lock]). 23 24%% =================================================================== 25%% Public API 26%% =================================================================== 27 28-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. 29init(State) -> 30 Provider = providers:create([{name, ?PROVIDER}, 31 {module, ?MODULE}, 32 {deps, ?DEPS}, 33 {bare, true}, 34 {example, "rebar3 ct"}, 35 {short_desc, "Run Common Tests."}, 36 {desc, "Run Common Tests."}, 37 {opts, ct_opts(State)}, 38 {profiles, [test]}]), 39 {ok, rebar_state:add_provider(State, Provider)}. 40 41-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. 42do(State) -> 43 setup_name(State), 44 Tests = prepare_tests(State), 45 case compile(State, Tests) of 46 %% successfully compiled apps 47 {ok, S} -> 48 {RawOpts, _} = rebar_state:command_parsed_args(S), 49 case proplists:get_value(compile_only, RawOpts, false) of 50 true -> 51 {ok, S}; 52 false -> 53 do(S, Tests) 54 end; 55 %% this should look like a compiler error, not a ct error 56 Error -> Error 57 end. 58 59do(State, Tests) -> 60 ?INFO("Running Common Test suites...", []), 61 rebar_paths:set_paths([deps, plugins], State), 62 63 %% Run ct provider prehooks 64 Providers = rebar_state:providers(State), 65 Cwd = rebar_dir:get_cwd(), 66 67 %% Run ct provider pre hooks for all project apps and top level project hooks 68 rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State), 69 70 case Tests of 71 {ok, T} -> 72 case run_tests(State, T) of 73 ok -> 74 %% Run ct provider post hooks for all project apps and top level project hooks 75 rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), 76 rebar_paths:set_paths([plugins, deps], State), 77 symlink_to_last_ct_logs(State, T), 78 {ok, State}; 79 Error -> 80 rebar_paths:set_paths([plugins, deps], State), 81 symlink_to_last_ct_logs(State, T), 82 Error 83 end; 84 Error -> 85 rebar_paths:set_paths([plugins, deps], State), 86 Error 87 end. 88 89run_tests(State, Opts) -> 90 T = translate_paths(State, Opts), 91 Opts1 = setup_logdir(State, T), 92 Opts2 = turn_off_auto_compile(Opts1), 93 ?DEBUG("Running tests with {ct_opts, ~p}.", [Opts2]), 94 {RawOpts, _} = rebar_state:command_parsed_args(State), 95 Result = case proplists:get_value(verbose, RawOpts, false) of 96 true -> run_test_verbose(Opts2); 97 false -> run_test_quiet(Opts2) 98 end, 99 ok = maybe_write_coverdata(State), 100 Result. 101 102-spec format_error(any()) -> iolist(). 103format_error({error, Reason}) -> 104 io_lib:format("Error running tests:~n ~p", [Reason]); 105format_error({error_running_tests, Reason}) -> 106 format_error({error, Reason}); 107format_error({failures_running_tests, {Failed, AutoSkipped}}) -> 108 io_lib:format("Failures occurred running tests: ~b", [Failed+AutoSkipped]); 109format_error({badconfig, {Msg, {Value, Key}}}) -> 110 io_lib:format(Msg, [Value, Key]); 111format_error({badconfig, Msg}) -> 112 io_lib:format(Msg, []); 113format_error({multiple_errors, Errors}) -> 114 io_lib:format(lists:concat(["Error running tests:"] ++ 115 lists:map(fun(Error) -> "~n " ++ Error end, Errors)), []); 116format_error({error_reading_testspec, Reason}) -> 117 io_lib:format("Error reading testspec: ~p", [Reason]). 118 119%% =================================================================== 120%% Internal functions 121%% =================================================================== 122 123%% @doc Tries to make the symlink `_build/<profile>/logs/last' to the `ct_run' directory 124%% of the last common test run. 125-spec symlink_to_last_ct_logs(rebar_state:t(), list()) -> ok. 126symlink_to_last_ct_logs(State, Opts) -> 127 LogDir = case proplists:get_value(logdir, Opts) of 128 undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); 129 Dir -> Dir 130 end, 131 {ok, Filenames} = file:list_dir(LogDir), 132 CtRunDirs = lists:filter(fun(S) -> re:run(S, "ct_run", [unicode]) /= nomatch end, Filenames), 133 case CtRunDirs of 134 [] -> 135 % If for some reason there are no such directories, we should not try to set up a link either. 136 ok; 137 _ -> 138 NewestDir = lists:last(lists:sort(CtRunDirs)), 139 Target = filename:join([LogDir, "last"]), 140 Existing = filename:join([LogDir, NewestDir]), 141 case rebar_file_utils:symlink_or_copy(Existing, Target) of 142 ok -> ok; 143 exists -> 144 %% in case the symlink already exists we remove it 145 %% and make a new updated one 146 rebar_file_utils:rm_rf(Target), 147 rebar_file_utils:symlink_or_copy(Existing, Target); 148 Reason -> ?DIAGNOSTIC("Warning, couldn't make a symlink to ~ts, reason: ~p.", [Target, Reason]) 149 end 150 end. 151 152setup_name(State) -> 153 {Long, Short, Opts} = rebar_dist_utils:find_options(State), 154 rebar_dist_utils:either(Long, Short, Opts). 155 156prepare_tests(State) -> 157 %% command line test options 158 CmdOpts = cmdopts(State), 159 %% rebar.config test options 160 CfgOpts = cfgopts(State), 161 ProjectApps = rebar_state:project_apps(State), 162 163 %% prioritize tests to run first trying any command line specified 164 %% tests falling back to tests specified in the config file finally 165 %% running a default set if no other tests are present 166 select_tests(State, ProjectApps, CmdOpts, CfgOpts). 167 168cmdopts(State) -> 169 {RawOpts, _} = rebar_state:command_parsed_args(State), 170 %% filter out opts common_test doesn't know about and convert 171 %% to ct acceptable forms 172 transform_retry(transform_opts(RawOpts, []), State). 173 174transform_opts([], Acc) -> lists:reverse(Acc); 175transform_opts([{dir, Dirs}|Rest], Acc) -> 176 transform_opts(Rest, [{dir, split_string(Dirs)}|Acc]); 177transform_opts([{suite, Suites}|Rest], Acc) -> 178 transform_opts(Rest, [{suite, split_string(Suites)}|Acc]); 179transform_opts([{group, Groups}|Rest], Acc) -> 180 transform_opts(Rest, [{group, transform_group(Groups)}|Acc]); 181transform_opts([{testcase, Cases}|Rest], Acc) -> 182 transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]); 183transform_opts([{config, Configs}|Rest], Acc) -> 184 transform_opts(Rest, [{config, split_string(Configs)}|Acc]); 185transform_opts([{spec, Specs}|Rest], Acc) -> 186 transform_opts(Rest, [{spec, split_string(Specs)}|Acc]); 187transform_opts([{include, Includes}|Rest], Acc) -> 188 transform_opts(Rest, [{include, split_string(Includes)}|Acc]); 189transform_opts([{logopts, LogOpts}|Rest], Acc) -> 190 transform_opts(Rest, [{logopts, lists:map(fun(P) -> list_to_atom(P) end, split_string(LogOpts))}|Acc]); 191transform_opts([{force_stop, "true"}|Rest], Acc) -> 192 transform_opts(Rest, [{force_stop, true}|Acc]); 193transform_opts([{force_stop, "false"}|Rest], Acc) -> 194 transform_opts(Rest, [{force_stop, false}|Acc]); 195transform_opts([{force_stop, "skip_rest"}|Rest], Acc) -> 196 transform_opts(Rest, [{force_stop, skip_rest}|Acc]); 197transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) -> 198 transform_opts(Rest, [{create_priv_dir, list_to_atom(CreatePrivDir)}|Acc]); 199%% drop cover from opts, ct doesn't care about it 200transform_opts([{cover, _}|Rest], Acc) -> 201 transform_opts(Rest, Acc); 202%% drop verbose from opts, ct doesn't care about it 203transform_opts([{verbose, _}|Rest], Acc) -> 204 transform_opts(Rest, Acc); 205%% drop fail_fast from opts, ct doesn't care about it 206transform_opts([{fail_fast, _}|Rest], Acc) -> 207 transform_opts(Rest, Acc); 208%% getopt should handle anything else 209transform_opts([Opt|Rest], Acc) -> 210 transform_opts(Rest, [Opt|Acc]). 211 212%% @private only retry if specified and if no other spec 213%% is given. 214transform_retry(Opts, State) -> 215 case proplists:get_value(retry, Opts, false) andalso 216 not is_any_defined([spec,dir,suite], Opts) of 217 false -> 218 Opts; 219 true -> 220 Path = filename:join([rebar_dir:base_dir(State), "logs", "retry.spec"]), 221 filelib:is_file(Path) andalso [{spec, Path}|Opts] 222 end. 223 224split_string(String) -> 225 rebar_string:lexemes(String, [$,]). 226 227transform_group(String) -> 228 case rebar_string:consult([$[, String, $], $.]) of 229 [Terms] when is_list(Terms) -> 230 Terms; 231 Terms when is_list(Terms) -> 232 Terms; 233 {error, _} -> 234 %% try a normal string split 235 split_string(String) 236 end. 237 238cfgopts(State) -> 239 case rebar_state:get(State, ct_opts, []) of 240 Opts when is_list(Opts) -> 241 ensure_opts(add_hooks(Opts, State), []); 242 Wrong -> 243 %% probably a single non list term 244 ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, ct_opts}}}) 245 end. 246 247ensure_opts([], Acc) -> lists:reverse(Acc); 248ensure_opts([{cover, _}|Rest], Acc) -> 249 ?WARN("Cover specs not supported. See https://www.rebar3.org/docs/testing/ct/", []), 250 ensure_opts(Rest, Acc); 251ensure_opts([{auto_compile, _}|Rest], Acc) -> 252 ?WARN("Auto compile not supported", []), 253 ensure_opts(Rest, Acc); 254ensure_opts([{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) -> 255 ensure_opts(Rest, [{suite, Suite}|Acc]); 256ensure_opts([{suite, Suite}|Rest], Acc) when is_atom(Suite) -> 257 ensure_opts(Rest, [{suite, atom_to_list(Suite)}|Acc]); 258ensure_opts([{suite, Suites}|Rest], Acc) -> 259 NewSuites = {suite, lists:map(fun(S) when is_atom(S) -> atom_to_list(S); 260 (S) when is_list(S) -> S 261 end, 262 Suites)}, 263 ensure_opts(Rest, [NewSuites|Acc]); 264ensure_opts([{K, V}|Rest], Acc) -> 265 ensure_opts(Rest, [{K, V}|Acc]); 266%% pass through other options, in case of things like config terms 267%% in `ct_opts` 268ensure_opts([V|Rest], Acc) -> 269 ensure_opts(Rest, [V|Acc]). 270 271add_hooks(Opts, State) -> 272 FailFast = case fails_fast(State) of 273 true -> [cth_fail_fast]; 274 false -> [] 275 end, 276 case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of 277 {false, _} -> 278 Opts; 279 {Other, false} -> 280 [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), 281 cth_retry] ++ FailFast} | Opts]; 282 {Other, {ct_hooks, Hooks}} -> 283 %% Make sure hooks are there once only and add wanted hooks that are not defined yet 284 ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), 285 cth_retry] ++ FailFast, 286 NewHooks = Hooks ++ [ReadableHook || 287 ReadableHook <- ReadableHooks, 288 not is_defined(ReadableHook, Hooks) 289 ], 290 lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks}) 291 end. 292 293is_defined(_Key, []) -> false; 294is_defined(Key, [Key | _Hs]) -> true; 295is_defined(Key, [{Key, _Opts} | _Hs]) -> true; 296is_defined(Key, [{Key, _Opts, _Prio} | _Hs]) -> true; 297is_defined(Key, [_ | Hs]) -> is_defined(Key, Hs). 298 299 300readable_shell_type(true) -> cth_readable_shell; 301readable_shell_type(compact) -> cth_readable_compact_shell. 302 303select_tests(_, _, _, {error, _} = Error) -> Error; 304select_tests(State, ProjectApps, CmdOpts, CfgOpts) -> 305 %% set application env if sys_config argument is provided 306 SysConfigs = sys_config_list(CmdOpts, CfgOpts), 307 Configs = lists:flatmap(fun(Filename) -> 308 rebar_file_utils:consult_config(State, Filename) 309 end, SysConfigs), 310 %% NB: load the applications (from user directories too) to support OTP < 17 311 %% to our best ability. 312 rebar_paths:set_paths([deps, plugins], State), 313 [application:load(Application) || Config <- Configs, {Application, _} <- Config], 314 rebar_utils:reread_config(Configs, [update_logger]), 315 316 Opts = merge_opts(CmdOpts,CfgOpts), 317 discover_tests(State, ProjectApps, Opts). 318 319%% Merge the option lists from command line and rebar.config: 320%% 321%% - Options set on the command line will replace the same options if 322%% set in rebar.config. 323%% 324%% - Special care is taken with options that select which tests to 325%% run - ANY such option on the command line will replace ALL such 326%% options in the config. 327%% 328%% Note that if 'spec' is given, common_test will ignore all 'dir', 329%% 'suite', 'group' and 'case', so there is no need to explicitly 330%% remove any options from the command line. 331%% 332%% All faulty combinations of options are also handled by 333%% common_test and are not taken into account here. 334merge_opts(CmdOpts0, CfgOpts0) -> 335 TestSelectOpts = [spec,dir,suite,group,testcase], 336 CmdOpts = lists:ukeysort(1, CmdOpts0), 337 CfgOpts1 = lists:ukeysort(1, CfgOpts0), 338 CfgOpts = case is_any_defined(TestSelectOpts,CmdOpts) of 339 false -> 340 CfgOpts1; 341 true -> 342 [Opt || Opt={K,_} <- CfgOpts1, 343 not lists:member(K,TestSelectOpts)] 344 end, 345 lists:ukeymerge(1, CmdOpts, CfgOpts). 346 347is_any_defined([Key|Keys],Opts) -> 348 proplists:is_defined(Key,Opts) orelse is_any_defined(Keys,Opts); 349is_any_defined([],_Opts) -> 350 false. 351 352sys_config_list(CmdOpts, CfgOpts) -> 353 CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), 354 case proplists:get_value(sys_config, CfgOpts, []) of 355 [H | _]=Configs when is_list(H) -> 356 Configs ++ CmdSysConfigs; 357 [] -> 358 CmdSysConfigs; 359 Configs -> 360 [Configs | CmdSysConfigs] 361 end. 362 363discover_tests(State, ProjectApps, Opts) -> 364 case is_any_defined([spec,dir,suite],Opts) of 365 %% no tests defined, try using `$APP/test` and `$ROOT/test` as dirs 366 false -> {ok, default_tests(State, ProjectApps) ++ Opts}; 367 true -> {ok, Opts} 368 end. 369 370default_tests(State, ProjectApps) -> 371 BareTest = filename:join([rebar_state:dir(State), "test"]), 372 F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, 373 AppTests = application_dirs(ProjectApps, []), 374 case filelib:is_dir(BareTest) andalso not lists:any(F, ProjectApps) of 375 %% `test` dir at root of project is already scheduled to be 376 %% included or `test` does not exist 377 false -> 378 %% The rest of the call-chain expects the list of tests to not be 379 %% empty, thus we drop the parameter in that case entirely. 380 case AppTests of 381 [] -> 382 []; 383 _ -> 384 [{dir, AppTests}] 385 end; 386 %% need to add `test` dir at root to dirs to be included 387 true -> 388 [{dir, AppTests ++ [BareTest]}] 389 end. 390 391application_dirs([], []) -> []; 392application_dirs([], Acc) -> lists:reverse(Acc); 393application_dirs([App|Rest], Acc) -> 394 TestDir = filename:join([rebar_app_info:dir(App), "test"]), 395 case filelib:is_dir(TestDir) of 396 true -> application_dirs(Rest, [TestDir|Acc]); 397 false -> application_dirs(Rest, Acc) 398 end. 399 400compile(State, {ok, _} = Tests) -> 401 %% inject `ct_first_files`, `ct_compile_opts` and `include` (from `ct_opts` 402 %% and command line options) into the applications to be compiled 403 case inject_ct_state(State, Tests) of 404 {ok, NewState} -> do_compile(NewState); 405 Error -> Error 406 end; 407%% maybe compile even in the face of errors? 408compile(_State, Error) -> Error. 409 410do_compile(State) -> 411 ?DEBUG("Re-compiling the project under the test profile with CT options injected...", []), 412 {ok, S} = rebar_prv_compile:do(State), 413 ok = maybe_cover_compile(S), 414 {ok, S}. 415 416inject_ct_state(State, {ok, Tests}) -> 417 Apps = rebar_state:project_apps(State), 418 case inject_ct_state(State, Tests, Apps, []) of 419 {ok, {NewState, ModdedApps}} -> 420 test_dirs(NewState, ModdedApps, Tests); 421 {error, _} = Error -> Error 422 end. 423 424inject_ct_state(State, Tests, [App|Rest], Acc) -> 425 case inject(rebar_app_info:opts(App), State, Tests) of 426 {error, _} = Error -> Error; 427 NewOpts -> 428 NewApp = rebar_app_info:opts(App, NewOpts), 429 inject_ct_state(State, Tests, Rest, [NewApp|Acc]) 430 end; 431inject_ct_state(State, Tests, [], Acc) -> 432 case inject(rebar_state:opts(State), State, Tests) of 433 {error, _} = Error -> Error; 434 NewOpts -> 435 {ok, {rebar_state:opts(State, NewOpts), lists:reverse(Acc)}} 436 end. 437 438opts(Opts, Key, Default) -> 439 case rebar_opts:get(Opts, Key, Default) of 440 Vs when is_list(Vs) -> Vs; 441 Wrong -> 442 ?PRV_ERROR({badconfig, {"Value `~p' of option `~p' must be a list", {Wrong, Key}}}) 443 end. 444 445inject(Opts, State, Tests) -> erl_opts(Opts, State, Tests). 446 447erl_opts(Opts, State, Tests) -> 448 %% append `ct_compile_opts` to app defined `erl_opts` 449 ErlOpts = opts(Opts, erl_opts, []), 450 CTOpts = opts(Opts, ct_compile_opts, []), 451 case add_transforms(append(CTOpts, ErlOpts), State) of 452 {error, _} = Error -> Error; 453 NewErlOpts -> first_files(rebar_opts:set(Opts, erl_opts, NewErlOpts), Tests) 454 end. 455 456first_files(Opts, Tests) -> 457 %% append `ct_first_files` to app defined `erl_first_files` 458 FirstFiles = opts(Opts, erl_first_files, []), 459 CTFirstFiles = opts(Opts, ct_first_files, []), 460 case append(CTFirstFiles, FirstFiles) of 461 {error, _} = Error -> Error; 462 NewFirstFiles -> include_files(rebar_opts:set(Opts, erl_first_files, NewFirstFiles), Tests) 463 end. 464 465include_files(Opts, Tests) -> 466 %% append include dirs from command line and `ct_opts` to app defined 467 %% `erl_opts` 468 ErlOpts = opts(Opts, erl_opts, []), 469 Includes = proplists:get_value(include, Tests, []), 470 Is = lists:map(fun(I) -> {i, I} end, Includes), 471 case append(Is, ErlOpts) of 472 {error, _} = Error -> Error; 473 NewIncludes -> ct_macro(rebar_opts:set(Opts, erl_opts, NewIncludes)) 474 end. 475 476ct_macro(Opts) -> 477 ErlOpts = opts(Opts, erl_opts, []), 478 NewOpts = safe_define_ct_macro(ErlOpts), 479 rebar_opts:set(Opts, erl_opts, NewOpts). 480 481safe_define_ct_macro(Opts) -> 482 %% defining a compile macro twice results in an exception so 483 %% make sure 'COMMON_TEST' is only defined once 484 case test_defined(Opts) of 485 true -> Opts; 486 false -> [{d, 'COMMON_TEST'}|Opts] 487 end. 488 489test_defined([{d, 'COMMON_TEST'}|_]) -> true; 490test_defined([{d, 'COMMON_TEST', true}|_]) -> true; 491test_defined([_|Rest]) -> test_defined(Rest); 492test_defined([]) -> false. 493 494append({error, _} = Error, _) -> Error; 495append(_, {error, _} = Error) -> Error; 496append(A, B) -> A ++ B. 497 498add_transforms(CTOpts, State) when is_list(CTOpts) -> 499 case readable(State) of 500 false -> 501 CTOpts; 502 Other when Other == true; Other == compact -> 503 ReadableTransform = [{parse_transform, cth_readable_transform}], 504 (CTOpts -- ReadableTransform) ++ ReadableTransform 505 end; 506add_transforms({error, _} = Error, _State) -> Error. 507 508readable(State) -> 509 {RawOpts, _} = rebar_state:command_parsed_args(State), 510 case proplists:get_value(readable, RawOpts) of 511 "true" -> true; 512 "false" -> false; 513 "compact" -> compact; 514 undefined -> rebar_state:get(State, ct_readable, compact) 515 end. 516 517fails_fast(State) -> 518 {RawOpts, _} = rebar_state:command_parsed_args(State), 519 proplists:get_value(fail_fast, RawOpts) == true. 520 521test_dirs(State, Apps, Opts) -> 522 case proplists:get_value(spec, Opts) of 523 undefined -> 524 case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of 525 {undefined, undefined} -> 526 {ok, rebar_state:project_apps(State, Apps)}; 527 {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites}); 528 {undefined, Dirs} -> set_compile_dirs(State, Apps, {dir, Dirs}); 529 {Suites, Dir} when is_integer(hd(Dir)) -> 530 set_compile_dirs(State, Apps, join(Suites, Dir)); 531 {Suites, [Dir]} when is_integer(hd(Dir)) -> 532 set_compile_dirs(State, Apps, join(Suites, Dir)); 533 {_Suites, _Dirs} -> {error, "Only a single directory may be specified when specifying suites"} 534 end; 535 Spec when is_integer(hd(Spec)) -> 536 spec_test_dirs(State, Apps, [Spec]); 537 Specs -> 538 spec_test_dirs(State, Apps, Specs) 539 end. 540 541spec_test_dirs(State, Apps, Specs0) -> 542 case get_dirs_from_specs(Specs0) of 543 {ok,{Specs,SuiteDirs}} -> 544 {State1,Apps1} = set_compile_dirs1(State, Apps, {dir, SuiteDirs}), 545 {State2,Apps2} = set_compile_dirs1(State1, Apps1, {spec, Specs}), 546 [maybe_copy_spec(State2,Apps2,S) || S <- Specs], 547 {ok, rebar_state:project_apps(State2, Apps2)}; 548 Error -> 549 Error 550 end. 551 552join(Suite, Dir) when is_integer(hd(Suite)) -> 553 {suite, [filename:join([Dir, Suite])]}; 554join(Suites, Dir) -> 555 {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}. 556 557set_compile_dirs(State, Apps, What) -> 558 {NewState,NewApps} = set_compile_dirs1(State, Apps, What), 559 {ok, rebar_state:project_apps(NewState, NewApps)}. 560 561set_compile_dirs1(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) -> 562 %% single directory 563 %% insert `Dir` into an app if relative, or the base state if not 564 %% app relative but relative to the root or not at all if outside 565 %% project scope 566 maybe_inject_test_dir(State, [], Apps, Dir); 567set_compile_dirs1(State, Apps, {dir, Dirs}) -> 568 %% multiple directories 569 F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, 570 lists:foldl(F, {State, Apps}, Dirs); 571set_compile_dirs1(State, Apps, {Type, Files}) when Type==spec; Type==suite -> 572 %% specs or suites with dir component 573 Dirs = find_file_dirs(Files), 574 F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end, 575 lists:foldl(F, {State, Apps}, Dirs). 576 577find_file_dirs(Files) -> 578 AllDirs = lists:map(fun(F) -> filename:dirname(filename:absname(F)) end, Files), 579 %% eliminate duplicates 580 lists:usort(AllDirs). 581 582maybe_inject_test_dir(State, AppAcc, [App|Rest], Dir) -> 583 case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of 584 {ok, []} -> 585 %% normal operation involves copying the entire directory a 586 %% suite exists in but if the suite is in the app root directory 587 %% the current compiler tries to compile all subdirs including priv 588 %% instead copy only files ending in `.erl' and directories 589 %% ending in `_SUITE_data' into the `_build/PROFILE/lib/APP' dir 590 ok = copy_bare_suites(Dir, rebar_app_info:out_dir(App)), 591 Opts = inject_test_dir(rebar_state:opts(State), rebar_app_info:out_dir(App)), 592 {rebar_state:opts(State, Opts), AppAcc ++ [App]}; 593 {ok, Path} -> 594 Opts = inject_test_dir(rebar_app_info:opts(App), Path), 595 {State, AppAcc ++ [rebar_app_info:opts(App, Opts)] ++ Rest}; 596 {error, badparent} -> 597 maybe_inject_test_dir(State, AppAcc ++ [App], Rest, Dir) 598 end; 599maybe_inject_test_dir(State, AppAcc, [], Dir) -> 600 case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of 601 {ok, []} -> 602 %% normal operation involves copying the entire directory a 603 %% suite exists in but if the suite is in the root directory 604 %% that results in a loop as we copy `_build' into itself 605 %% instead copy only files ending in `.erl' and directories 606 %% ending in `_SUITE_data' in the `_build/PROFILE/extras' dir 607 ExtrasDir = filename:join([rebar_dir:base_dir(State), "extras"]), 608 ok = copy_bare_suites(Dir, ExtrasDir), 609 Opts = inject_test_dir(rebar_state:opts(State), ExtrasDir), 610 {rebar_state:opts(State, Opts), AppAcc}; 611 {ok, Path} -> 612 Opts = inject_test_dir(rebar_state:opts(State), Path), 613 {rebar_state:opts(State, Opts), AppAcc}; 614 {error, badparent} -> 615 {State, AppAcc} 616 end. 617 618copy_bare_suites(From, To) -> 619 filelib:ensure_dir(filename:join([To, "dummy.txt"])), 620 SrcFiles = rebar_utils:find_files(From, ".*\\.[e|h]rl\$", false), 621 DataDirs = lists:filter(fun filelib:is_dir/1, 622 filelib:wildcard(filename:join([From, "*_SUITE_data"]))), 623 ok = rebar_file_utils:cp_r(SrcFiles, To), 624 rebar_file_utils:cp_r(DataDirs, To). 625 626maybe_copy_spec(State, [App|Apps], Spec) -> 627 case rebar_file_utils:path_from_ancestor(filename:dirname(Spec), rebar_app_info:dir(App)) of 628 {ok, []} -> 629 ok = rebar_file_utils:cp_r([Spec],rebar_app_info:out_dir(App)); 630 {ok,_} -> 631 ok; 632 {error,badparent} -> 633 maybe_copy_spec(State, Apps, Spec) 634 end; 635maybe_copy_spec(State, [], Spec) -> 636 case rebar_file_utils:path_from_ancestor(filename:dirname(Spec), rebar_state:dir(State)) of 637 {ok, []} -> 638 ExtrasDir = filename:join([rebar_dir:base_dir(State), "extras"]), 639 ok = rebar_file_utils:cp_r([Spec],ExtrasDir); 640 _R -> 641 ok 642 end. 643 644inject_test_dir(Opts, Dir) -> 645 %% append specified test targets to app defined `extra_src_dirs` 646 ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []), 647 rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]). 648 649get_dirs_from_specs(Specs) -> 650 case get_tests_from_specs(Specs) of 651 {ok,Tests} -> 652 {SpecLists,NodeRunSkipLists} = lists:unzip(Tests), 653 SpecList = lists:append(SpecLists), 654 NodeRunSkipList = lists:append(NodeRunSkipLists), 655 RunList = lists:append([R || {_,R,_} <- NodeRunSkipList]), 656 DirList = [element(1,R) || R <- RunList], 657 {ok,{SpecList,DirList}}; 658 {error,Reason} -> 659 {error,{?MODULE,{error_reading_testspec,Reason}}} 660 end. 661 662get_tests_from_specs(Specs) -> 663 _ = ct_testspec:module_info(), % make sure ct_testspec is loaded 664 case erlang:function_exported(ct_testspec,get_tests,1) of 665 true -> 666 ct_testspec:get_tests(Specs); 667 false -> 668 case ct_testspec:collect_tests_from_file(Specs,true) of 669 Tests when is_list(Tests) -> 670 {ok,[{S,ct_testspec:prepare_tests(R)} || {S,R} <- Tests]}; 671 Error -> 672 Error 673 end 674 end. 675 676translate_paths(State, Opts) -> 677 case proplists:get_value(spec, Opts) of 678 undefined -> 679 case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of 680 {_Suites, undefined} -> translate_paths(State, suite, Opts, []); 681 {undefined, _Dirs} -> translate_paths(State, dir, Opts, []); 682 %% both dirs and suites are defined, only translate dir paths 683 _ -> translate_paths(State, dir, Opts, []) 684 end; 685 _Specs -> 686 translate_paths(State, spec, Opts, []) 687 end. 688 689translate_paths(_State, _Type, [], Acc) -> lists:reverse(Acc); 690translate_paths(State, Type, [{Type, Val}|Rest], Acc) when is_integer(hd(Val)) -> 691 %% single file or dir 692 translate_paths(State, Type, [{Type, [Val]}|Rest], Acc); 693translate_paths(State, Type, [{Type, Files}|Rest], Acc) -> 694 Apps = rebar_state:project_apps(State), 695 New = {Type, lists:map(fun(File) -> translate(State, Apps, File) end, Files)}, 696 translate_paths(State, Type, Rest, [New|Acc]); 697translate_paths(State, Type, [Test|Rest], Acc) -> 698 translate_paths(State, Type, Rest, [Test|Acc]). 699 700translate(State, [App|Rest], Path) -> 701 case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of 702 {ok, P} -> filename:join([rebar_app_info:out_dir(App), P]); 703 {error, badparent} -> translate(State, Rest, Path) 704 end; 705translate(State, [], Path) -> 706 case rebar_file_utils:path_from_ancestor(Path, rebar_state:dir(State)) of 707 {ok, P} -> filename:join([rebar_dir:base_dir(State), "extras", P]); 708 %% not relative, leave as is 709 {error, badparent} -> Path 710 end. 711 712-spec handle_keep_logs(file:filename(), pos_integer()) -> ok. 713handle_keep_logs(LogDir, N) -> 714 case file:list_dir(LogDir) of 715 {ok, Filenames} -> 716 Dirs = lists:filter(fun(File) -> 717 filelib:is_dir(filename:join([LogDir, File])) 718 end, Filenames) -- ["last"], %% we ignore the symlink as we later handle it 719 case Dirs of 720 %% first time running the tests, there are no logs to delete 721 [] -> ok; 722 %% during the next run we would crash because of keep_logs 723 _ when length(Dirs) >= N -> 724 SortedDirs = lists:reverse(lists:sort(Dirs)), 725 %% sort the log dirs and keep the N - 1 newest 726 {_Keep, Discard} = lists:split(N - 1, SortedDirs), 727 ?DEBUG("Removing the following directories because keep_logs is in ct_opts: ~p", [Discard]), 728 [rebar_file_utils:rm_rf(filename:join([LogDir, Dir])) || Dir <- Discard], 729 ok; 730 %% we still dont have enough log run directories as to crash 731 _ -> ok 732 end; 733 _ -> ok 734 end. 735 736setup_logdir(State, Opts) -> 737 Logdir = case proplists:get_value(logdir, Opts) of 738 undefined -> filename:join([rebar_dir:base_dir(State), "logs"]); 739 Dir -> Dir 740 end, 741 filelib:ensure_dir(filename:join([Logdir, "dummy.beam"])), 742 case proplists:get_value(keep_logs, Opts) of 743 all -> ok; 744 undefined -> ok; 745 N -> handle_keep_logs(Logdir, N) 746 end, 747 [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)]. 748 749turn_off_auto_compile(Opts) -> 750 [{auto_compile, false}|lists:keydelete(auto_compile, 1, Opts)]. 751 752run_test_verbose(Opts) -> handle_results(ct:run_test(Opts)). 753 754run_test_quiet(Opts) -> 755 Pid = self(), 756 Ref = erlang:make_ref(), 757 LogDir = proplists:get_value(logdir, Opts), 758 {_, Monitor} = erlang:spawn_monitor(fun() -> 759 {ok, F} = file:open(filename:join([LogDir, "ct.latest.log"]), 760 [write]), 761 true = group_leader(F, self()), 762 Pid ! {Ref, ct:run_test(Opts)} 763 end), 764 receive 765 {Ref, Result} -> handle_quiet_results(Opts, Result); 766 {'DOWN', Monitor, _, _, Reason} -> handle_results(?PRV_ERROR(Reason)) 767 end. 768 769handle_results(Results) when is_list(Results) -> 770 Result = lists:foldl(fun sum_results/2, {0, 0, {0,0}}, Results), 771 handle_results(Result); 772handle_results({_, Failed, {_, AutoSkipped}}) 773 when Failed > 0 orelse AutoSkipped > 0 -> 774 ?PRV_ERROR({failures_running_tests, {Failed, AutoSkipped}}); 775handle_results({error, Reason}) -> 776 ?PRV_ERROR({error_running_tests, Reason}); 777handle_results(_) -> 778 ok. 779 780sum_results({Passed, Failed, {UserSkipped, AutoSkipped}}, 781 {Passed2, Failed2, {UserSkipped2, AutoSkipped2}}) -> 782 {Passed+Passed2, Failed+Failed2, 783 {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}; 784sum_results(_, {error, Reason}) -> 785 {error, Reason}; 786sum_results(Unknown, _) -> 787 {error, Unknown}. 788 789handle_quiet_results(_, {error, _} = Result) -> 790 handle_results(Result); 791handle_quiet_results(CTOpts, Results) when is_list(Results) -> 792 _ = [format_result(Result) || Result <- Results], 793 case handle_results(Results) of 794 ?PRV_ERROR({failures_running_tests, _}) = Error -> 795 LogDir = proplists:get_value(logdir, CTOpts), 796 Index = filename:join([LogDir, "index.html"]), 797 ?CONSOLE("Results written to ~p.", [Index]), 798 Error; 799 Other -> 800 Other 801 end; 802handle_quiet_results(CTOpts, Result) -> 803 handle_quiet_results(CTOpts, [Result]). 804 805format_result({Passed, 0, {0, 0}}) -> 806 ?CONSOLE("All ~p tests passed.", [Passed]); 807format_result({Passed, Failed, Skipped}) -> 808 Format = [format_failed(Failed), format_skipped(Skipped), 809 format_passed(Passed)], 810 ?CONSOLE("~ts", [Format]); 811format_result(_Unknown) -> 812 %% Happens when CT itself encounters a bug 813 ok. 814 815format_failed(0) -> 816 []; 817format_failed(Failed) -> 818 io_lib:format("Failed ~p tests. ", [Failed]). 819 820format_passed(Passed) -> 821 io_lib:format("Passed ~p tests. ", [Passed]). 822 823format_skipped({0, 0}) -> 824 []; 825format_skipped({User, Auto}) -> 826 io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]). 827 828maybe_cover_compile(State) -> 829 {RawOpts, _} = rebar_state:command_parsed_args(State), 830 State1 = case proplists:get_value(cover, RawOpts, false) of 831 true -> rebar_state:set(State, cover_enabled, true); 832 false -> State 833 end, 834 rebar_prv_cover:maybe_cover_compile(State1). 835 836maybe_write_coverdata(State) -> 837 {RawOpts, _} = rebar_state:command_parsed_args(State), 838 State1 = case proplists:get_value(cover, RawOpts, false) of 839 true -> rebar_state:set(State, cover_enabled, true); 840 false -> State 841 end, 842 Name = proplists:get_value(cover_export_name, RawOpts, ?PROVIDER), 843 rebar_prv_cover:maybe_write_coverdata(State1, Name). 844 845ct_opts(_State) -> 846 [{dir, undefined, "dir", string, help(dir)}, %% comma-separated list 847 {suite, undefined, "suite", string, help(suite)}, %% comma-separated list 848 {group, undefined, "group", string, help(group)}, %% comma-separated list 849 {testcase, undefined, "case", string, help(testcase)}, %% comma-separated list 850 {label, undefined, "label", string, help(label)}, %% String 851 {config, undefined, "config", string, help(config)}, %% comma-separated list 852 {spec, undefined, "spec", string, help(spec)}, %% comma-separated list 853 {join_specs, undefined, "join_specs", boolean, help(join_specs)}, 854 {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool 855 {logdir, undefined, "logdir", string, help(logdir)}, %% dir 856 {logopts, undefined, "logopts", string, help(logopts)}, %% comma-separated list 857 {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer 858 {cover, $c, "cover", {boolean, false}, help(cover)}, 859 {cover_export_name, undefined, "cover_export_name", string, help(cover_export_name)}, 860 {repeat, undefined, "repeat", integer, help(repeat)}, %% integer 861 {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS 862 {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS] 863 {force_stop, undefined, "force_stop", string, help(force_stop)}, %% String 864 {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Boolean 865 {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% String 866 {decrypt_key, undefined, "decrypt_key", string, help(decrypt_key)}, %% String 867 {decrypt_file, undefined, "decrypt_file", string, help(decrypt_file)}, %% String 868 {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% Boolean 869 {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiple_timetraps)}, %% Integer 870 {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, 871 {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, 872 {include, undefined, "include", string, help(include)}, 873 {readable, undefined, "readable", string, help(readable)}, 874 {verbose, $v, "verbose", boolean, help(verbose)}, 875 {name, undefined, "name", atom, help(name)}, 876 {sname, undefined, "sname", atom, help(sname)}, 877 {setcookie, undefined, "setcookie", atom, help(setcookie)}, 878 {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list 879 {compile_only, undefined, "compile_only", boolean, help(compile_only)}, 880 {retry, undefined, "retry", boolean, help(retry)}, 881 {fail_fast, undefined, "fail_fast", {boolean, false}, help(fail_fast)} 882 ]. 883 884help(compile_only) -> 885 "Compile modules in the project with the test configuration but do not run the tests"; 886help(dir) -> 887 "List of additional directories containing test suites"; 888help(suite) -> 889 "List of test suites to run"; 890help(group) -> 891 "List of test groups to run"; 892help(testcase) -> 893 "List of test cases to run"; 894help(label) -> 895 "Test label"; 896help(config) -> 897 "List of config files"; 898help(spec) -> 899 "List of test specifications"; 900help(join_specs) -> 901 "Merge all test specifications and perform a single test run"; 902help(sys_config) -> 903 "List of application config files"; 904help(allow_user_terms) -> 905 "Allow user defined config values in config files"; 906help(logdir) -> 907 "Log folder"; 908help(logopts) -> 909 "Options for common test logging"; 910help(verbosity) -> 911 "Verbosity"; 912help(cover) -> 913 "Generate cover data"; 914help(cover_export_name) -> 915 "Base name of the coverdata file to write"; 916help(repeat) -> 917 "How often to repeat tests"; 918help(duration) -> 919 "Max runtime (format: HHMMSS)"; 920help(until) -> 921 "Run until (format: HHMMSS)"; 922help(force_stop) -> 923 "Force stop on test timeout (true | false | skip_rest)"; 924help(basic_html) -> 925 "Show basic HTML"; 926help(stylesheet) -> 927 "CSS stylesheet to apply to html output"; 928help(decrypt_key) -> 929 "Path to key for decrypting config"; 930help(decrypt_file) -> 931 "Path to file containing key for decrypting config"; 932help(abort_if_missing_suites) -> 933 "Abort if suites are missing"; 934help(multiply_timetraps) -> 935 "Multiply timetraps"; 936help(scale_timetraps) -> 937 "Scale timetraps"; 938help(create_priv_dir) -> 939 "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)"; 940help(include) -> 941 "Directories containing additional include files"; 942help(readable) -> 943 "Shows test case names and only displays logs to shell on failures (true | compact | false)"; 944help(verbose) -> 945 "Verbose output"; 946help(name) -> 947 "Gives a long name to the node"; 948help(sname) -> 949 "Gives a short name to the node"; 950help(setcookie) -> 951 "Sets the cookie if the node is distributed"; 952help(retry) -> 953 "Experimental feature. If any specification for previously failing test is found, runs them."; 954help(fail_fast) -> 955 "Experimental feature. If any test fails, the run is aborted. Since common test does not " 956 "support this natively, we abort the rebar3 run on a failure. This May break CT's disk logging and " 957 "other rebar3 features."; 958help(_) -> 959 "". 960