1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2009-2020. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20-module(ct_tc_repeat_SUITE).
21
22-compile(export_all).
23
24-include_lib("common_test/include/ct.hrl").
25-include_lib("common_test/include/ct_event.hrl").
26
27-define(eh, ct_test_support_eh).
28
29%%--------------------------------------------------------------------
30%% TEST SERVER CALLBACK FUNCTIONS
31%%--------------------------------------------------------------------
32
33%%--------------------------------------------------------------------
34%% Description: Since Common Test starts another Test Server
35%% instance, the tests need to be performed on a separate node (or
36%% there will be clashes with logging processes etc).
37%%--------------------------------------------------------------------
38init_per_suite(Config) ->
39    DataDir = ?config(data_dir, Config),
40    ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]).
41
42end_per_suite(Config) ->
43    ct_test_support:end_per_suite(Config).
44
45init_per_testcase(TestCase, Config) ->
46    ct_test_support:init_per_testcase(TestCase, Config).
47
48end_per_testcase(TestCase, Config) ->
49    ct_test_support:end_per_testcase(TestCase, Config).
50
51
52suite() ->
53    [{timetrap,{minutes,1}}].
54
55all() ->
56    all(suite).
57
58all(suite) ->
59      [
60       repeat,
61       repeat_parallel_until_ok,
62       repeat_parallel_until_fail,
63       repeat_sequence_until_ok,
64       repeat_sequence_until_fail,
65       pick_one_test_from_group,
66       pick_one_test_from_subgroup
67      ].
68
69
70%%--------------------------------------------------------------------
71%% TEST CASES
72%%--------------------------------------------------------------------
73
74%%%-----------------------------------------------------------------
75%%%
76%% Test post_groups and post_all hook callbacks, introduced by OTP-14746
77repeat(Config) ->
78    ok = do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [], [], Config).
79
80repeat_parallel_until_ok(Config) ->
81    {error,{{illegal_combination,{parallel,repeat_until_ok}},_}} =
82        do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_ok}],
83                [], Config, 1, []).
84
85repeat_parallel_until_fail(Config) ->
86    {error,{{illegal_combination,{parallel,repeat_until_fail}},_}} =
87        do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_fail}],
88                [], Config, 1, []).
89
90repeat_sequence_until_ok(Config) ->
91    {error,{{illegal_combination,{sequence,repeat_until_ok}},_}} =
92        do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_ok}],
93                [], Config, 1, []).
94
95repeat_sequence_until_fail(Config) ->
96    {error,{{illegal_combination,{sequence,repeat_until_fail}},_}} =
97        do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_fail}],
98                [], Config, 1, []).
99
100pick_one_test_from_group(Config) ->
101    do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_mixed},{testcase,tc2}],
102            [], Config, 1, []).
103
104pick_one_test_from_subgroup(Config) ->
105    do_test(?FUNCTION_NAME, "tc_repeat_SUITE",
106            [{group,[[g_mixed,subgroup]]},{testcase,tc2}],
107            [], Config, 1, []).
108
109
110%%%-----------------------------------------------------------------
111%%% HELP FUNCTIONS
112%%%-----------------------------------------------------------------
113
114do_test(Tag, Suite, WTT, CTHs, Config) ->
115    do_test(Tag, Suite, WTT, CTHs, Config, 2, []).
116
117do_test(Tag, Suite0, WTT, CTHs, Config, EC, ExtraOpts) ->
118    DataDir = ?config(data_dir, Config),
119    Suite = filename:join([DataDir,Suite0]),
120    {Opts,ERPid} =
121        setup([{suite,Suite}|WTT]++[{ct_hooks,CTHs},{label,Tag}|ExtraOpts],
122              Config),
123    Res = ct_test_support:run(Opts, Config),
124    Events = ct_test_support:get_events(ERPid, Config),
125    %% io:format("~p~n",[Events]),
126
127    ct_test_support:log_events(Tag,
128			       reformat(Events, ?eh),
129			       ?config(priv_dir, Config),
130			       Opts),
131
132    TestEvents = events_to_check(Tag, EC),
133    ok = ct_test_support:verify_events(TestEvents, Events, Config),
134    Res.
135
136setup(Test, Config) ->
137    Opts0 = ct_test_support:get_opts(Config),
138    Level = ?config(trace_level, Config),
139    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
140    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
141    ERPid = ct_test_support:start_event_receiver(Config),
142    {Opts,ERPid}.
143
144reformat(Events, EH) ->
145    ct_test_support:reformat(Events, EH).
146
147gen_config(Name,KeyVals,Config) ->
148    PrivDir = ?config(priv_dir,Config),
149    File = filename:join(PrivDir,atom_to_list(Name)++".cfg"),
150    ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}])
151                               || {Key,Value} <- KeyVals]),
152    File.
153
154%%%-----------------------------------------------------------------
155%%% TEST EVENTS
156%%%-----------------------------------------------------------------
157events_to_check(Test) ->
158    %% 2 tests (ct:run_test + script_start) is default
159    events_to_check(Test, 2).
160
161events_to_check(_, 0) ->
162    [];
163events_to_check(Test, N) ->
164    test_events(Test) ++ events_to_check(Test, N-1).
165
166test_events(repeat) ->
167    S = tc_repeat_SUITE,
168    [
169     {?eh,start_logging,{'DEF','RUNDIR'}},
170     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
171     {?eh,start_info,{1,1,unknown}},
172
173     %% tc1, {repeat,2}
174     {?eh,tc_start,{S,tc1}},
175     {?eh,tc_done,{S,tc1,ok}},
176     {?eh,test_stats,{1,0,{0,0}}},
177     {?eh,tc_start,{S,tc1}},
178     {?eh,tc_done,{S,tc1,ok}},
179     {?eh,test_stats,{2,0,{0,0}}},
180     %% tc2, {repeat_until_ok,3}
181     {?eh,tc_start,{S,tc2}},
182     {?eh,tc_done,{S,tc2,ok}},
183     {?eh,test_stats,{3,0,{0,0}}},
184     %% tc3, {repeat_until_ok,3}
185     {?eh,tc_start,{S,tc3}},
186     {?eh,tc_done,{tc_repeat_SUITE,tc3,
187                   {failed,{error,{test_case_failed,always_fail}}}}},
188     {?eh,test_stats,{3,1,{0,0}}},
189     {?eh,tc_start,{S,tc3}},
190     {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}},
191     {?eh,test_stats,{3,2,{0,0}}},
192     {?eh,tc_start,{S,tc3}},
193     {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}},
194     {?eh,test_stats,{3,3,{0,0}}},
195     %% tc4, {repeat_until_fail,3}
196     {?eh,tc_start,{S,tc4}},
197     {?eh,tc_done,{S,tc4,ok}},
198     {?eh,test_stats,{4,3,{0,0}}},
199     {?eh,tc_start,{S,tc4}},
200     {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}},
201     {?eh,test_stats,{4,4,{0,0}}},
202     %% g, tc1, {repeat,2}
203     {?eh,tc_start,{S,{init_per_group,g,[]}}},
204     {?eh,tc_done,{S,{init_per_group,g,[]},ok}},
205     {?eh,tc_start,{S,tc1}},
206     {?eh,tc_done,{S,tc1,ok}},
207     {?eh,test_stats,{5,4,{0,0}}},
208     {?eh,tc_start,{S,tc1}},
209     {?eh,tc_done,{S,tc1,ok}},
210     {?eh,test_stats,{6,4,{0,0}}},
211     {?eh,tc_start,{S,{end_per_group,g,[]}}},
212     {?eh,tc_done,{S,{end_per_group,g,[]},ok}},
213     %% g_until_ok, tc2, {repeat_until_ok,3}
214     {?eh,tc_start,{S,{init_per_group,g_until_ok,[]}}},
215     {?eh,tc_done,{S,{init_per_group,g_until_ok,[]},ok}},
216     {?eh,tc_start,{S,tc2}},
217     {?eh,tc_done,{S,tc2,ok}},
218     {?eh,test_stats,{7,4,{0,0}}},
219     {?eh,tc_start,{S,{end_per_group,g_until_ok,[]}}},
220     {?eh,tc_done,{S,{end_per_group,g_until_ok,[]},ok}},
221     %% g_until_fail, tc4, {repeat_until_fail,3}
222     {?eh,tc_start,{S,{init_per_group,g_until_fail,[]}}},
223     {?eh,tc_done,{S,{init_per_group,g_until_fail,[]},ok}},
224     {?eh,tc_start,{S,tc4}},
225     {?eh,tc_done,{S,tc4,ok}},
226     {?eh,test_stats,{8,4,{0,0}}},
227     {?eh,tc_start,{S,tc4}},
228     {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}},
229     {?eh,test_stats,{8,5,{0,0}}},
230     {?eh,tc_start,{S,{end_per_group,g_until_fail,[]}}},
231     {?eh,tc_done,{S,{end_per_group,g_until_fail,[]},ok}},
232     %% g, parallel, tc1, {repeat,2}
233     {parallel,
234      [{?eh,tc_start,{S,{init_per_group,g,[parallel]}}},
235       {?eh,tc_done,{S,{init_per_group,g,[parallel]},ok}},
236       {?eh,tc_start,{S,tc1}},
237       {?eh,tc_done,{S,tc1,ok}},
238       {?eh,test_stats,{9,5,{0,0}}},
239       {?eh,tc_start,{S,tc1}},
240       {?eh,tc_done,{S,tc1,ok}},
241       {?eh,test_stats,{10,5,{0,0}}},
242       {?eh,tc_start,{S,{end_per_group,g,[parallel]}}},
243       {?eh,tc_done,{S,{end_per_group,g,[parallel]},ok}}]},
244     %% g, sequence, tc1, {repeat,2}
245     {?eh,tc_start,{S,{init_per_group,g,[sequence]}}},
246     {?eh,tc_done,{S,{init_per_group,g,[sequence]},ok}},
247     {?eh,tc_start,{S,tc1}},
248     {?eh,tc_done,{S,tc1,ok}},
249     {?eh,test_stats,{11,5,{0,0}}},
250     {?eh,tc_start,{S,tc1}},
251     {?eh,tc_done,{S,tc1,ok}},
252     {?eh,test_stats,{12,5,{0,0}}},
253     {?eh,tc_start,{S,{end_per_group,g,[sequence]}}},
254     {?eh,tc_done,{S,{end_per_group,g,[sequence]},ok}},
255     %% g_sequence_skip_rest,
256     {?eh,tc_start,{S,{init_per_group,g_mixed,[sequence]}}},
257     {?eh,tc_done,{S,{init_per_group,g_mixed,[sequence]},ok}},
258     {?eh,tc_start,{S,tc1}},
259     {?eh,tc_done,{S,tc1,ok}},
260     {?eh,test_stats,{13,5,{0,0}}},
261     {?eh,tc_start,{S,tc1}},
262     {?eh,tc_done,{S,tc1,ok}},
263     {?eh,test_stats,{14,5,{0,0}}},
264     {?eh,tc_start,{S,tc4}},
265     {?eh,tc_done,{S,tc4,ok}},
266     {?eh,test_stats,{15,5,{0,0}}},
267     {?eh,tc_start,{S,tc4}},
268     {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}},
269     {?eh,test_stats,{15,6,{0,0}}},
270     %% ----> fail in sequence, so skip rest
271     {?eh,tc_auto_skip,{S,{tc4,g_mixed}, % last of current repeat tc4
272                        {failed,{tc_repeat_SUITE,tc4}}}},
273     {?eh,test_stats,{15,6,{0,1}}},
274     {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1
275                        {failed,{tc_repeat_SUITE,tc4}}}},
276     {?eh,test_stats,{15,6,{0,2}}},
277     {?eh,tc_auto_skip,{S,{tc1,g}, % group g, tc1, {repeat,2}
278                        {failed,{tc_repeat_SUITE,tc4}}}},
279     {?eh,test_stats,{15,6,{0,3}}},
280     {?eh,tc_auto_skip,{S,{tc1,subgroup}, % subgroup, single tc1
281                        {failed,{tc_repeat_SUITE,tc4}}}},
282     {?eh,test_stats,{15,6,{0,4}}},
283     {?eh,tc_auto_skip,{S,{tc2,subgroup}, % subgroup, tc2, {repeat,2}
284                        {failed,{tc_repeat_SUITE,tc4}}}},
285     {?eh,test_stats,{15,6,{0,5}}},
286     {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % tc2, {repeat,2}
287                        {failed,{tc_repeat_SUITE,tc4}}}},
288     {?eh,test_stats,{15,6,{0,6}}},
289     {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % single tc2
290                        {failed,{tc_repeat_SUITE,tc4}}}},
291     {?eh,test_stats,{15,6,{0,7}}},
292     {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % tc1, {repeat,2}
293                        {failed,{tc_repeat_SUITE,tc4}}}},
294     {?eh,test_stats,{15,6,{0,8}}},
295     {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1
296                        {failed,{tc_repeat_SUITE,tc4}}}},
297     {?eh,test_stats,{15,6,{0,9}}},
298     {?eh,tc_start,{S,{end_per_group,g_mixed,'_'}}},
299     {?eh,tc_done,{S,{end_per_group,g_mixed,'_'},ok}},
300     %% done
301     {?eh,test_done,{'DEF','STOP_TIME'}},
302     {?eh,stop_logging,[]}
303    ];
304
305test_events(repeat_parallel_until_ok) ->
306    [
307     {?eh,start_logging,{'DEF','RUNDIR'}},
308     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
309     {?eh,tc_start,{'_',{init_per_group,g_parallel_until_ok,[parallel]}}},
310     {?eh,tc_done,{'_',{init_per_group,g_parallel_until_ok,[parallel]},ok}},
311     {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_ok}},'_'}},
312     {?eh,test_done,{'DEF','STOP_TIME'}},
313     {?eh,stop_logging,[]}
314    ];
315
316test_events(repeat_parallel_until_fail) ->
317    [
318     {?eh,start_logging,{'DEF','RUNDIR'}},
319     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
320     {?eh,tc_start,{'_',{init_per_group,g_parallel_until_fail,[parallel]}}},
321     {?eh,tc_done,{'_',{init_per_group,g_parallel_until_fail,[parallel]},ok}},
322     {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_fail}},'_'}},
323     {?eh,test_done,{'DEF','STOP_TIME'}},
324     {?eh,stop_logging,[]}
325    ];
326
327test_events(repeat_sequence_until_ok) ->
328    [
329     {?eh,start_logging,{'DEF','RUNDIR'}},
330     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
331     {?eh,tc_start,{'_',{init_per_group,g_sequence_until_ok,[sequence]}}},
332     {?eh,tc_done,{'_',{init_per_group,g_sequence_until_ok,[sequence]},ok}},
333     {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_ok}},'_'}},
334     {?eh,test_done,{'DEF','STOP_TIME'}},
335     {?eh,stop_logging,[]}
336    ];
337
338test_events(repeat_sequence_until_fail) ->
339    [
340     {?eh,start_logging,{'DEF','RUNDIR'}},
341     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
342     {?eh,tc_start,{'_',{init_per_group,g_sequence_until_fail,[sequence]}}},
343     {?eh,tc_done,{'_',{init_per_group,g_sequence_until_fail,[sequence]},ok}},
344     {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_fail}},'_'}},
345     {?eh,test_done,{'DEF','STOP_TIME'}},
346     {?eh,stop_logging,[]}
347    ];
348
349test_events(pick_one_test_from_group) ->
350    [
351     {?eh,start_logging,{'DEF','RUNDIR'}},
352     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
353     {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}},
354     {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}},
355     {negative,
356      {?eh,tc_start,{'_',tc1}},
357      {?eh,tc_start,{'_',tc2}}}, % single tc2
358     {?eh,tc_done,{'_',tc2,ok}},
359     {?eh,tc_start,{'_',tc2}}, % tc2, {repeat,2}
360     {?eh,tc_done,{'_',tc2,ok}},
361     {?eh,tc_start,{'_',tc2}},
362     {?eh,tc_done,{'_',tc2,ok}},
363     {negative,
364      {?eh,tc_start,{'_',tc1}},
365      {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}},
366     {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}},
367     {?eh,test_done,{'DEF','STOP_TIME'}},
368     {?eh,stop_logging,[]}
369    ];
370
371test_events(pick_one_test_from_subgroup) ->
372    [
373     {?eh,start_logging,{'DEF','RUNDIR'}},
374     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
375     {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}},
376     {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}},
377     {negative,
378      {?eh,tc_start,{'_',tc2}},
379      {?eh,tc_start,{'_',{init_per_group,subgroup,[]}}}},
380     {?eh,tc_done,{'_',{init_per_group,subgroup,[]},ok}},
381     {negative,
382      {?eh,tc_start,{'_',tc1}},
383      {?eh,tc_start,{'_',tc2}}}, % tc2, {repeat,2}
384     {?eh,tc_done,{'_',tc2,ok}},
385     {?eh,tc_start,{'_',tc2}},
386     {?eh,tc_done,{'_',tc2,ok}},
387     {negative,
388      {?eh,tc_start,{'_',tc1}},
389      {?eh,tc_start,{'_',{end_per_group,subgroup,[]}}}},
390     {?eh,tc_done,{'_',{end_per_group,subgroup,[]},ok}},
391     {negative,
392      {?eh,tc_start,{'_',tc2}},
393      {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}},
394     {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}},
395     {?eh,test_done,{'DEF','STOP_TIME'}},
396     {?eh,stop_logging,[]}
397    ];
398
399test_events(ok) ->
400    ok.
401
402%% test events help functions
403contains(List) ->
404    fun(Proplist) when is_list(Proplist) ->
405	    contains(List,Proplist)
406    end.
407
408contains([{not_in_order,List}|T],Rest) ->
409    contains_parallel(List,Rest),
410    contains(T,Rest);
411contains([{Ele,Pos}|T] = L,[H|T2]) ->
412    case element(Pos,H) of
413	Ele ->
414	    contains(T,T2);
415	_ ->
416	    contains(L,T2)
417    end;
418contains([Ele|T],[{Ele,_}|T2])->
419    contains(T,T2);
420contains([Ele|T],[Ele|T2])->
421    contains(T,T2);
422contains(List,[_|T]) ->
423    contains(List,T);
424contains([],_) ->
425    match.
426
427contains_parallel([Key | T], Elems) ->
428    contains([Key],Elems),
429    contains_parallel(T,Elems);
430contains_parallel([],_Elems) ->
431    match.
432
433not_contains(List) ->
434    fun(Proplist) when is_list(Proplist) ->
435	    [] = [Ele || {Ele,_} <- Proplist,
436			 Test <- List,
437			 Test =:= Ele]
438    end.
439