1%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
2%% ex: ts=4 sw=4 et
3%% -------------------------------------------------------------------
4%%
5%% rebar: Erlang Build Tools
6%%
7%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com)
8%%
9%% Permission is hereby granted, free of charge, to any person obtaining a copy
10%% of this software and associated documentation files (the "Software"), to deal
11%% in the Software without restriction, including without limitation the rights
12%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13%% copies of the Software, and to permit persons to whom the Software is
14%% furnished to do so, subject to the following conditions:
15%%
16%% The above copyright notice and this permission notice shall be included in
17%% all copies or substantial portions of the Software.
18%%
19%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25%% THE SOFTWARE.
26%% -------------------------------------------------------------------
27%%
28%% Targets:
29%% test - run common test suites in ./test
30%% int_test - run suites in ./int_test
31%% perf_test - run suites inm ./perf_test
32%%
33%% Global options:
34%% verbose=1 - show output from the common_test run as it goes
35%% suites="foo,bar" - run <test>/foo_SUITE and <test>/bar_SUITE
36%% case="mycase" - run individual test case foo_SUITE:mycase
37%% -------------------------------------------------------------------
38-module(rebar_ct).
39
40-export([ct/2]).
41
42%% for internal use only
43-export([info/2]).
44
45-include("rebar.hrl").
46
47%% ===================================================================
48%% Public API
49%% ===================================================================
50
51ct(Config, File) ->
52    TestDir = rebar_config:get_local(Config, ct_dir, "test"),
53    LogDir = rebar_config:get_local(Config, ct_log_dir, "logs"),
54    run_test_if_present(TestDir, LogDir, Config, File).
55
56%% ===================================================================
57%% Internal functions
58%% ===================================================================
59
60info(help, ct) ->
61    ?CONSOLE(
62       "Run common_test suites.~n"
63       "~n"
64       "Valid rebar.config options:~n"
65       "  ~p~n"
66       "  ~p~n"
67       "  ~p~n"
68       "  ~p~n"
69       "  ~p~n"
70       "Valid command line options:~n"
71       "  suites=Suite1,Suite2,...,SuiteN~n"
72       "      - run Suite1_SUITE, Suite2_SUITE, ..., SuiteN_SUITE~n"
73       "      in the test folder.~n"
74       "  groups=Group1,Group2,...,GroupN~n"
75       "      - run test groups Group1, Group2, ..., GroupN of specified suites.~n"
76       "  cases=Case1,Case2,...,CaseM~n"
77       "      - run test cases Case1, Case2, ..., CaseN of specified suites.~n"
78       "  case=\"mycase\" - run individual test case Suite1_SUITE:mycase.~n"
79       "      This option is deprecated and remains for backward compability.~n"
80       "      It is recommended to use 'cases' instead.~n",
81       [
82        {ct_dir, "itest"},
83        {ct_log_dir, "test/logs"},
84        {ct_extra_params, "-boot start_sasl -s myapp"},
85        {ct_use_short_names, true},
86        {ct_search_specs_from_test_dir, false}
87       ]).
88
89run_test_if_present(TestDir, LogDir, Config, File) ->
90    case filelib:is_dir(TestDir) of
91        false ->
92            ?WARN("~s directory not present - skipping\n", [TestDir]),
93            ok;
94        true ->
95            case filelib:wildcard(TestDir ++ "/*_SUITE.{beam,erl}") of
96                [] ->
97                    ?WARN("~s directory present, but no common_test"
98                          ++ " SUITES - skipping\n", [TestDir]),
99                    ok;
100                _ ->
101                    try
102                        run_test(TestDir, LogDir, Config, File)
103                    catch
104                        throw:skip ->
105                            ok
106                    end
107            end
108    end.
109
110run_test(TestDir, LogDir, Config, _File) ->
111    {Cmd, RawLog} = make_cmd(TestDir, LogDir, Config),
112    ?DEBUG("ct_run cmd:~n~p~n", [Cmd]),
113    clear_log(LogDir, RawLog),
114    Output = case rebar_log:is_verbose(Config) of
115                 false ->
116                     " >> " ++ RawLog ++ " 2>&1";
117                 true ->
118                 case os:type() of
119                   {win32, nt} ->
120                     " >> " ++ RawLog ++ " 2>&1";
121                   _ ->
122                     " 2>&1 | tee -a " ++ RawLog
123                 end
124             end,
125
126    ShOpts = [{env,[{"TESTDIR", TestDir}]}, return_on_error],
127    case rebar_utils:sh(Cmd ++ Output, ShOpts) of
128        {ok,_} ->
129            %% in older versions of ct_run, this could have been a failure
130            %% that returned a non-0 code. Check for that!
131            check_success_log(Config, RawLog);
132        {error,Res} ->
133            %% In newer ct_run versions, this may be a sign of a good compile
134            %% that failed cases. In older version, it's a worse error.
135            check_fail_log(Config, RawLog, Cmd ++ Output, Res)
136    end.
137
138clear_log(LogDir, RawLog) ->
139    case filelib:ensure_dir(filename:join(LogDir, "index.html")) of
140        ok ->
141            NowStr = rebar_utils:now_str(),
142            LogHeader = "--- Test run on " ++ NowStr ++ " ---\n",
143            ok = file:write_file(RawLog, LogHeader);
144        {error, Reason} ->
145            ?ERROR("Could not create log dir - ~p\n", [Reason]),
146            ?FAIL
147    end.
148
149%% calling ct with erl does not return non-zero on failure - have to check
150%% log results
151check_success_log(Config, RawLog) ->
152    check_log(Config, RawLog, fun(Msg) -> ?CONSOLE("DONE.\n~s\n", [Msg]) end).
153
154-type err_handler() :: fun((string()) -> no_return()).
155-spec failure_logger(string(), {integer(), string()}) -> err_handler().
156failure_logger(Command, {Rc, Output}) ->
157    fun(_Msg) ->
158            ?ABORT("~s failed with error: ~w and output:~n~s~n",
159                   [Command, Rc, Output])
160    end.
161
162check_fail_log(Config, RawLog, Command, Result) ->
163    check_log(Config, RawLog, failure_logger(Command, Result)).
164
165check_log(Config,RawLogFilename,Fun) ->
166    %% read the file and split into a list separated by newlines
167    {ok, RawLog} = file:read_file(RawLogFilename),
168    Msg = string:tokens(binary_to_list(RawLog), "\n"),
169    %% now filter out all the list entries that do not have test
170    %% completion strings
171    CompleteRuns = lists:filter(fun(M) ->
172                                  string:str(M, "TEST COMPLETE") =/= 0
173                                end, Msg),
174    MakeFailed = lists:filter(fun(M) ->
175                                  string:str(M, "{error,make_failed}") =/= 0
176                              end, Msg),
177    %% the run has failed if at least one of the tests failed
178    RunFailed = lists:foldl(fun(M, Acc) ->
179                              %% the "0 failed" string must be present for
180                              %% the test to be considered successful
181                              TestFailed = string:str(M, "0 failed") =:= 0,
182                              TestFailed orelse Acc
183                            end, false, CompleteRuns),
184    if
185        MakeFailed =/= [] ->
186            show_log(Config, RawLog),
187            ?ERROR("Building tests failed\n",[]),
188            ?FAIL;
189
190        RunFailed ->
191            show_log(Config, RawLog),
192            ?ERROR("One or more tests failed\n",[]),
193            ?FAIL;
194
195        true ->
196            Fun(Msg)
197    end.
198
199
200%% Show the log if it hasn't already been shown because verbose was on
201show_log(Config, RawLog) ->
202    ?CONSOLE("Showing log\n", []),
203    case rebar_log:is_verbose(Config) of
204        false ->
205            ?CONSOLE("~s", [RawLog]);
206        true ->
207            ok
208    end.
209
210make_cmd(TestDir, RawLogDir, Config) ->
211    Cwd = rebar_utils:get_cwd(),
212    LogDir = filename:join(Cwd, RawLogDir),
213    EbinDir = filename:absname(filename:join(Cwd, "ebin")),
214    IncludeDir = filename:join(Cwd, "include"),
215    Include = case filelib:is_dir(IncludeDir) of
216                  true ->
217                      " -include \"" ++ IncludeDir ++ "\"";
218                  false ->
219                      ""
220              end,
221
222    %% Check for the availability of ct_run; if we can't find it, generate a
223    %% warning and use the old school, less reliable approach to running CT.
224    BaseCmd = case os:find_executable("ct_run") of
225                  false ->
226                      "erl -noshell -s ct_run script_start -s erlang halt";
227                  _ ->
228                      "ct_run -noshell"
229              end,
230
231    %% Add the code path of the rebar process to the code path. This
232    %% includes the dependencies in the code path. The directories
233    %% that are part of the root Erlang install are filtered out to
234    %% avoid duplication
235    R = code:root_dir(),
236    NonLibCodeDirs = [P || P <- code:get_path(), not lists:prefix(R, P)],
237    CodeDirs = [io_lib:format("\"~s\"", [Dir]) ||
238                   Dir <- [EbinDir|NonLibCodeDirs]],
239    CodePathString = string:join(CodeDirs, " "),
240    Cmd = case get_ct_specs(Config, search_ct_specs_from(Cwd, TestDir, Config)) of
241              undefined ->
242                  ?FMT("~s"
243                       " -pa ~s"
244                       " ~s"
245                       " ~s"
246                       " -logdir \"~s\""
247                       " -env TEST_DIR \"~s\"",
248                       [BaseCmd,
249                        CodePathString,
250                        Include,
251                        build_name(Config),
252                        LogDir,
253                        filename:join(Cwd, TestDir)]) ++
254                      get_cover_config(Config, Cwd) ++
255                      get_ct_config_file(TestDir) ++
256                      get_suites(Config, TestDir) ++
257                      get_groups(Config) ++
258                      get_cases(Config) ++
259                      get_extra_params(Config) ++
260                      get_config_file(TestDir);
261              SpecFlags ->
262                  ?FMT("~s"
263                       " -pa ~s"
264                       " ~s"
265                       " ~s"
266                       " -logdir \"~s\""
267                       " -env TEST_DIR \"~s\"",
268                       [BaseCmd,
269                        CodePathString,
270                        Include,
271                        build_name(Config),
272                        LogDir,
273                        filename:join(Cwd, TestDir)]) ++
274                      SpecFlags ++
275                      get_cover_config(Config, Cwd) ++
276                      get_extra_params(Config)
277          end,
278    RawLog = filename:join(LogDir, "raw.log"),
279    {Cmd, RawLog}.
280
281search_ct_specs_from(Cwd, TestDir, Config) ->
282    case rebar_config:get_local(Config, ct_search_specs_from_test_dir, false) of
283        true -> filename:join(Cwd, TestDir);
284        false ->
285          Cwd
286    end.
287
288build_name(Config) ->
289    %% generate a unique name for our test node, we want
290    %% to make sure the odds of name clashing are low
291    Random = integer_to_list(crypto:rand_uniform(0, 10000)),
292    case rebar_config:get_local(Config, ct_use_short_names, false) of
293        true -> "-sname test" ++ Random;
294        false -> " -name test" ++ Random ++ "@" ++ net_adm:localhost()
295    end.
296
297get_extra_params(Config) ->
298    case rebar_config:get_local(Config, ct_extra_params, undefined) of
299        undefined ->
300            "";
301        Defined ->
302            " " ++ Defined
303    end.
304
305get_ct_specs(Config, Cwd) ->
306    case collect_glob(Config, Cwd, ".*\.test\.spec\$") of
307        [] -> undefined;
308        [Spec] ->
309            " -spec " ++ Spec;
310        Specs ->
311            " -spec " ++
312                lists:flatten([io_lib:format("~s ", [Spec]) || Spec <- Specs])
313    end.
314
315get_cover_config(Config, Cwd) ->
316    case rebar_config:get_local(Config, cover_enabled, false) of
317        false ->
318            "";
319        true ->
320            case collect_glob(Config, Cwd, ".*cover\.spec\$") of
321                [] ->
322                    ?DEBUG("No cover spec found: ~s~n", [Cwd]),
323                    "";
324                [Spec] ->
325                    ?DEBUG("Found cover file ~s~n", [Spec]),
326                    " -cover " ++ Spec;
327                Specs ->
328                    ?ABORT("Multiple cover specs found: ~p~n", [Specs])
329            end
330    end.
331
332collect_glob(Config, Cwd, Glob) ->
333    {true, Deps} = rebar_deps:get_deps_dir(Config),
334    DepsDir = filename:basename(Deps),
335    CwdParts = filename:split(Cwd),
336    filelib:fold_files(
337      Cwd,
338      Glob,
339      true,
340      fun(F, Acc) ->
341              %% Ignore any specs under the deps/ directory. Do this pulling
342              %% the dirname off the F and then splitting it into a list.
343              Parts = filename:split(filename:dirname(F)),
344              Parts2 = remove_common_prefix(Parts, CwdParts),
345              case lists:member(DepsDir, Parts2) of
346                  true ->
347                      %% There is a directory named "deps" in path
348                      Acc;
349                  false ->
350                      %% No "deps" directory in path
351                      [F | Acc]
352              end
353      end,
354      []).
355
356remove_common_prefix([H1|T1], [H1|T2]) ->
357    remove_common_prefix(T1, T2);
358remove_common_prefix(L1, _) ->
359    L1.
360
361get_ct_config_file(TestDir) ->
362    Config = filename:join(TestDir, "test.config"),
363    case filelib:is_regular(Config) of
364        false ->
365            " ";
366        true ->
367            " -ct_config " ++ Config
368    end.
369
370get_config_file(TestDir) ->
371    Config = filename:join(TestDir, "app.config"),
372    case filelib:is_regular(Config) of
373        false ->
374            " ";
375        true ->
376            " -erl_args -config " ++ Config
377    end.
378
379get_suites(Config, TestDir) ->
380    case get_suites(Config) of
381        undefined ->
382            " -dir " ++ TestDir;
383        Suites ->
384            Suites1 = [find_suite_path(Suite, TestDir) || Suite <- Suites],
385            string:join([" -suite"] ++ Suites1, " ")
386    end.
387
388get_suites(Config) ->
389    case rebar_config:get_global(Config, suites, undefined) of
390        undefined ->
391            %% The option 'suite' is deprecated and remains
392            %% for backward compatibility.
393            %% It is recommended to use 'suites' instead.
394            case get_deprecated_global(Config, suite, suites) of
395                undefined ->
396                    undefined;
397                Suite ->
398                    [Suite]
399            end;
400        Suites ->
401            string:tokens(Suites, ",")
402    end.
403
404find_suite_path(Suite, TestDir) ->
405    Path = filename:join(TestDir, Suite ++ "_SUITE.erl"),
406    case filelib:is_regular(Path) of
407        false ->
408            ?WARN("Suite ~s not found\n", [Suite]),
409            %% Note - this throw is caught in run_test_if_present/3;
410            %% this solution was easier than refactoring the entire module.
411            throw(skip);
412        true ->
413            Path
414    end.
415
416get_groups(Config) ->
417    case rebar_config:get_global(Config, groups, undefined) of
418        undefined ->
419            %% The option 'group' was added only for consistency
420            %% because there are options 'suite' and 'case'.
421            case get_deprecated_global(Config, group, groups) of
422                undefined ->
423                    "";
424                Group ->
425                    " -group " ++ Group
426            end;
427        Groups ->
428            Groups1 = string:tokens(Groups, ","),
429            string:join([" -group"] ++ Groups1, " ")
430    end.
431
432get_cases(Config) ->
433    case rebar_config:get_global(Config, cases, undefined) of
434        undefined ->
435            %% The option 'case' is deprecated and remains
436            %% for backward compatibility.
437            %% It is recommended to use 'cases' instead.
438            case get_deprecated_global(Config, 'case', cases) of
439                undefined ->
440                    "";
441                Case ->
442                    " -case " ++ Case
443            end;
444        Cases ->
445            Cases1 = string:tokens(Cases, ","),
446            string:join([" -case"] ++ Cases1, " ")
447    end.
448
449get_deprecated_global(Config, OldOpt, NewOpt) ->
450    rebar_utils:get_deprecated_global(
451      Config, OldOpt, NewOpt, undefined, "in the future").
452
453