1%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- 2%% ex: ts=4 sw=4 et 3%% ------------------------------------------------------------------- 4%% 5%% rebar: Erlang Build Tools 6%% 7%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com) 8%% 9%% Permission is hereby granted, free of charge, to any person obtaining a copy 10%% of this software and associated documentation files (the "Software"), to deal 11%% in the Software without restriction, including without limitation the rights 12%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13%% copies of the Software, and to permit persons to whom the Software is 14%% furnished to do so, subject to the following conditions: 15%% 16%% The above copyright notice and this permission notice shall be included in 17%% all copies or substantial portions of the Software. 18%% 19%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 25%% THE SOFTWARE. 26%% ------------------------------------------------------------------- 27%% 28%% Targets: 29%% test - run common test suites in ./test 30%% int_test - run suites in ./int_test 31%% perf_test - run suites inm ./perf_test 32%% 33%% Global options: 34%% verbose=1 - show output from the common_test run as it goes 35%% suites="foo,bar" - run <test>/foo_SUITE and <test>/bar_SUITE 36%% case="mycase" - run individual test case foo_SUITE:mycase 37%% ------------------------------------------------------------------- 38-module(rebar_ct). 39 40-export([ct/2]). 41 42%% for internal use only 43-export([info/2]). 44 45-include("rebar.hrl"). 46 47%% =================================================================== 48%% Public API 49%% =================================================================== 50 51ct(Config, File) -> 52 TestDir = rebar_config:get_local(Config, ct_dir, "test"), 53 LogDir = rebar_config:get_local(Config, ct_log_dir, "logs"), 54 run_test_if_present(TestDir, LogDir, Config, File). 55 56%% =================================================================== 57%% Internal functions 58%% =================================================================== 59 60info(help, ct) -> 61 ?CONSOLE( 62 "Run common_test suites.~n" 63 "~n" 64 "Valid rebar.config options:~n" 65 " ~p~n" 66 " ~p~n" 67 " ~p~n" 68 " ~p~n" 69 " ~p~n" 70 "Valid command line options:~n" 71 " suites=Suite1,Suite2,...,SuiteN~n" 72 " - run Suite1_SUITE, Suite2_SUITE, ..., SuiteN_SUITE~n" 73 " in the test folder.~n" 74 " groups=Group1,Group2,...,GroupN~n" 75 " - run test groups Group1, Group2, ..., GroupN of specified suites.~n" 76 " cases=Case1,Case2,...,CaseM~n" 77 " - run test cases Case1, Case2, ..., CaseN of specified suites.~n" 78 " case=\"mycase\" - run individual test case Suite1_SUITE:mycase.~n" 79 " This option is deprecated and remains for backward compability.~n" 80 " It is recommended to use 'cases' instead.~n", 81 [ 82 {ct_dir, "itest"}, 83 {ct_log_dir, "test/logs"}, 84 {ct_extra_params, "-boot start_sasl -s myapp"}, 85 {ct_use_short_names, true}, 86 {ct_search_specs_from_test_dir, false} 87 ]). 88 89run_test_if_present(TestDir, LogDir, Config, File) -> 90 case filelib:is_dir(TestDir) of 91 false -> 92 ?WARN("~s directory not present - skipping\n", [TestDir]), 93 ok; 94 true -> 95 case filelib:wildcard(TestDir ++ "/*_SUITE.{beam,erl}") of 96 [] -> 97 ?WARN("~s directory present, but no common_test" 98 ++ " SUITES - skipping\n", [TestDir]), 99 ok; 100 _ -> 101 try 102 run_test(TestDir, LogDir, Config, File) 103 catch 104 throw:skip -> 105 ok 106 end 107 end 108 end. 109 110run_test(TestDir, LogDir, Config, _File) -> 111 {Cmd, RawLog} = make_cmd(TestDir, LogDir, Config), 112 ?DEBUG("ct_run cmd:~n~p~n", [Cmd]), 113 clear_log(LogDir, RawLog), 114 Output = case rebar_log:is_verbose(Config) of 115 false -> 116 " >> " ++ RawLog ++ " 2>&1"; 117 true -> 118 case os:type() of 119 {win32, nt} -> 120 " >> " ++ RawLog ++ " 2>&1"; 121 _ -> 122 " 2>&1 | tee -a " ++ RawLog 123 end 124 end, 125 126 ShOpts = [{env,[{"TESTDIR", TestDir}]}, return_on_error], 127 case rebar_utils:sh(Cmd ++ Output, ShOpts) of 128 {ok,_} -> 129 %% in older versions of ct_run, this could have been a failure 130 %% that returned a non-0 code. Check for that! 131 check_success_log(Config, RawLog); 132 {error,Res} -> 133 %% In newer ct_run versions, this may be a sign of a good compile 134 %% that failed cases. In older version, it's a worse error. 135 check_fail_log(Config, RawLog, Cmd ++ Output, Res) 136 end. 137 138clear_log(LogDir, RawLog) -> 139 case filelib:ensure_dir(filename:join(LogDir, "index.html")) of 140 ok -> 141 NowStr = rebar_utils:now_str(), 142 LogHeader = "--- Test run on " ++ NowStr ++ " ---\n", 143 ok = file:write_file(RawLog, LogHeader); 144 {error, Reason} -> 145 ?ERROR("Could not create log dir - ~p\n", [Reason]), 146 ?FAIL 147 end. 148 149%% calling ct with erl does not return non-zero on failure - have to check 150%% log results 151check_success_log(Config, RawLog) -> 152 check_log(Config, RawLog, fun(Msg) -> ?CONSOLE("DONE.\n~s\n", [Msg]) end). 153 154-type err_handler() :: fun((string()) -> no_return()). 155-spec failure_logger(string(), {integer(), string()}) -> err_handler(). 156failure_logger(Command, {Rc, Output}) -> 157 fun(_Msg) -> 158 ?ABORT("~s failed with error: ~w and output:~n~s~n", 159 [Command, Rc, Output]) 160 end. 161 162check_fail_log(Config, RawLog, Command, Result) -> 163 check_log(Config, RawLog, failure_logger(Command, Result)). 164 165check_log(Config,RawLogFilename,Fun) -> 166 %% read the file and split into a list separated by newlines 167 {ok, RawLog} = file:read_file(RawLogFilename), 168 Msg = string:tokens(binary_to_list(RawLog), "\n"), 169 %% now filter out all the list entries that do not have test 170 %% completion strings 171 CompleteRuns = lists:filter(fun(M) -> 172 string:str(M, "TEST COMPLETE") =/= 0 173 end, Msg), 174 MakeFailed = lists:filter(fun(M) -> 175 string:str(M, "{error,make_failed}") =/= 0 176 end, Msg), 177 %% the run has failed if at least one of the tests failed 178 RunFailed = lists:foldl(fun(M, Acc) -> 179 %% the "0 failed" string must be present for 180 %% the test to be considered successful 181 TestFailed = string:str(M, "0 failed") =:= 0, 182 TestFailed orelse Acc 183 end, false, CompleteRuns), 184 if 185 MakeFailed =/= [] -> 186 show_log(Config, RawLog), 187 ?ERROR("Building tests failed\n",[]), 188 ?FAIL; 189 190 RunFailed -> 191 show_log(Config, RawLog), 192 ?ERROR("One or more tests failed\n",[]), 193 ?FAIL; 194 195 true -> 196 Fun(Msg) 197 end. 198 199 200%% Show the log if it hasn't already been shown because verbose was on 201show_log(Config, RawLog) -> 202 ?CONSOLE("Showing log\n", []), 203 case rebar_log:is_verbose(Config) of 204 false -> 205 ?CONSOLE("~s", [RawLog]); 206 true -> 207 ok 208 end. 209 210make_cmd(TestDir, RawLogDir, Config) -> 211 Cwd = rebar_utils:get_cwd(), 212 LogDir = filename:join(Cwd, RawLogDir), 213 EbinDir = filename:absname(filename:join(Cwd, "ebin")), 214 IncludeDir = filename:join(Cwd, "include"), 215 Include = case filelib:is_dir(IncludeDir) of 216 true -> 217 " -include \"" ++ IncludeDir ++ "\""; 218 false -> 219 "" 220 end, 221 222 %% Check for the availability of ct_run; if we can't find it, generate a 223 %% warning and use the old school, less reliable approach to running CT. 224 BaseCmd = case os:find_executable("ct_run") of 225 false -> 226 "erl -noshell -s ct_run script_start -s erlang halt"; 227 _ -> 228 "ct_run -noshell" 229 end, 230 231 %% Add the code path of the rebar process to the code path. This 232 %% includes the dependencies in the code path. The directories 233 %% that are part of the root Erlang install are filtered out to 234 %% avoid duplication 235 R = code:root_dir(), 236 NonLibCodeDirs = [P || P <- code:get_path(), not lists:prefix(R, P)], 237 CodeDirs = [io_lib:format("\"~s\"", [Dir]) || 238 Dir <- [EbinDir|NonLibCodeDirs]], 239 CodePathString = string:join(CodeDirs, " "), 240 Cmd = case get_ct_specs(Config, search_ct_specs_from(Cwd, TestDir, Config)) of 241 undefined -> 242 ?FMT("~s" 243 " -pa ~s" 244 " ~s" 245 " ~s" 246 " -logdir \"~s\"" 247 " -env TEST_DIR \"~s\"", 248 [BaseCmd, 249 CodePathString, 250 Include, 251 build_name(Config), 252 LogDir, 253 filename:join(Cwd, TestDir)]) ++ 254 get_cover_config(Config, Cwd) ++ 255 get_ct_config_file(TestDir) ++ 256 get_suites(Config, TestDir) ++ 257 get_groups(Config) ++ 258 get_cases(Config) ++ 259 get_extra_params(Config) ++ 260 get_config_file(TestDir); 261 SpecFlags -> 262 ?FMT("~s" 263 " -pa ~s" 264 " ~s" 265 " ~s" 266 " -logdir \"~s\"" 267 " -env TEST_DIR \"~s\"", 268 [BaseCmd, 269 CodePathString, 270 Include, 271 build_name(Config), 272 LogDir, 273 filename:join(Cwd, TestDir)]) ++ 274 SpecFlags ++ 275 get_cover_config(Config, Cwd) ++ 276 get_extra_params(Config) 277 end, 278 RawLog = filename:join(LogDir, "raw.log"), 279 {Cmd, RawLog}. 280 281search_ct_specs_from(Cwd, TestDir, Config) -> 282 case rebar_config:get_local(Config, ct_search_specs_from_test_dir, false) of 283 true -> filename:join(Cwd, TestDir); 284 false -> 285 Cwd 286 end. 287 288build_name(Config) -> 289 %% generate a unique name for our test node, we want 290 %% to make sure the odds of name clashing are low 291 Random = integer_to_list(crypto:rand_uniform(0, 10000)), 292 case rebar_config:get_local(Config, ct_use_short_names, false) of 293 true -> "-sname test" ++ Random; 294 false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost() 295 end. 296 297get_extra_params(Config) -> 298 case rebar_config:get_local(Config, ct_extra_params, undefined) of 299 undefined -> 300 ""; 301 Defined -> 302 " " ++ Defined 303 end. 304 305get_ct_specs(Config, Cwd) -> 306 case collect_glob(Config, Cwd, ".*\.test\.spec\$") of 307 [] -> undefined; 308 [Spec] -> 309 " -spec " ++ Spec; 310 Specs -> 311 " -spec " ++ 312 lists:flatten([io_lib:format("~s ", [Spec]) || Spec <- Specs]) 313 end. 314 315get_cover_config(Config, Cwd) -> 316 case rebar_config:get_local(Config, cover_enabled, false) of 317 false -> 318 ""; 319 true -> 320 case collect_glob(Config, Cwd, ".*cover\.spec\$") of 321 [] -> 322 ?DEBUG("No cover spec found: ~s~n", [Cwd]), 323 ""; 324 [Spec] -> 325 ?DEBUG("Found cover file ~s~n", [Spec]), 326 " -cover " ++ Spec; 327 Specs -> 328 ?ABORT("Multiple cover specs found: ~p~n", [Specs]) 329 end 330 end. 331 332collect_glob(Config, Cwd, Glob) -> 333 {true, Deps} = rebar_deps:get_deps_dir(Config), 334 DepsDir = filename:basename(Deps), 335 CwdParts = filename:split(Cwd), 336 filelib:fold_files( 337 Cwd, 338 Glob, 339 true, 340 fun(F, Acc) -> 341 %% Ignore any specs under the deps/ directory. Do this pulling 342 %% the dirname off the F and then splitting it into a list. 343 Parts = filename:split(filename:dirname(F)), 344 Parts2 = remove_common_prefix(Parts, CwdParts), 345 case lists:member(DepsDir, Parts2) of 346 true -> 347 %% There is a directory named "deps" in path 348 Acc; 349 false -> 350 %% No "deps" directory in path 351 [F | Acc] 352 end 353 end, 354 []). 355 356remove_common_prefix([H1|T1], [H1|T2]) -> 357 remove_common_prefix(T1, T2); 358remove_common_prefix(L1, _) -> 359 L1. 360 361get_ct_config_file(TestDir) -> 362 Config = filename:join(TestDir, "test.config"), 363 case filelib:is_regular(Config) of 364 false -> 365 " "; 366 true -> 367 " -ct_config " ++ Config 368 end. 369 370get_config_file(TestDir) -> 371 Config = filename:join(TestDir, "app.config"), 372 case filelib:is_regular(Config) of 373 false -> 374 " "; 375 true -> 376 " -erl_args -config " ++ Config 377 end. 378 379get_suites(Config, TestDir) -> 380 case get_suites(Config) of 381 undefined -> 382 " -dir " ++ TestDir; 383 Suites -> 384 Suites1 = [find_suite_path(Suite, TestDir) || Suite <- Suites], 385 string:join([" -suite"] ++ Suites1, " ") 386 end. 387 388get_suites(Config) -> 389 case rebar_config:get_global(Config, suites, undefined) of 390 undefined -> 391 %% The option 'suite' is deprecated and remains 392 %% for backward compatibility. 393 %% It is recommended to use 'suites' instead. 394 case get_deprecated_global(Config, suite, suites) of 395 undefined -> 396 undefined; 397 Suite -> 398 [Suite] 399 end; 400 Suites -> 401 string:tokens(Suites, ",") 402 end. 403 404find_suite_path(Suite, TestDir) -> 405 Path = filename:join(TestDir, Suite ++ "_SUITE.erl"), 406 case filelib:is_regular(Path) of 407 false -> 408 ?WARN("Suite ~s not found\n", [Suite]), 409 %% Note - this throw is caught in run_test_if_present/3; 410 %% this solution was easier than refactoring the entire module. 411 throw(skip); 412 true -> 413 Path 414 end. 415 416get_groups(Config) -> 417 case rebar_config:get_global(Config, groups, undefined) of 418 undefined -> 419 %% The option 'group' was added only for consistency 420 %% because there are options 'suite' and 'case'. 421 case get_deprecated_global(Config, group, groups) of 422 undefined -> 423 ""; 424 Group -> 425 " -group " ++ Group 426 end; 427 Groups -> 428 Groups1 = string:tokens(Groups, ","), 429 string:join([" -group"] ++ Groups1, " ") 430 end. 431 432get_cases(Config) -> 433 case rebar_config:get_global(Config, cases, undefined) of 434 undefined -> 435 %% The option 'case' is deprecated and remains 436 %% for backward compatibility. 437 %% It is recommended to use 'cases' instead. 438 case get_deprecated_global(Config, 'case', cases) of 439 undefined -> 440 ""; 441 Case -> 442 " -case " ++ Case 443 end; 444 Cases -> 445 Cases1 = string:tokens(Cases, ","), 446 string:join([" -case"] ++ Cases1, " ") 447 end. 448 449get_deprecated_global(Config, OldOpt, NewOpt) -> 450 rebar_utils:get_deprecated_global( 451 Config, OldOpt, NewOpt, undefined, "in the future"). 452 453