1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2009-2020. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20-module(ct_util_SUITE).
21
22-compile(export_all).
23
24-include_lib("common_test/include/ct.hrl").
25-include_lib("common_test/include/ct_event.hrl").
26
27-define(eh, ct_test_support_eh).
28
29%%--------------------------------------------------------------------
30%% TEST SERVER CALLBACK FUNCTIONS
31%%--------------------------------------------------------------------
32
33%%--------------------------------------------------------------------
34%% Description: Since Common Test starts another Test Server
35%% instance, the tests need to be performed on a separate node (or
36%% there will be clashes with logging processes etc).
37%%--------------------------------------------------------------------
38init_per_suite(Config) ->
39    DataDir = ?config(data_dir, Config),
40    CTHs = filelib:wildcard(filename:join(DataDir,"*_cth.erl")),
41    io:format("CTHs: ~p",[CTHs]),
42    [io:format("Compiling ~p: ~p",
43	    [FileName,compile:file(FileName,[{outdir,DataDir},debug_info])]) ||
44	FileName <- CTHs],
45    ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]).
46
47end_per_suite(Config) ->
48    ct_test_support:end_per_suite(Config).
49
50init_per_testcase(TestCase, Config) ->
51    ct_test_support:init_per_testcase(TestCase, Config).
52
53end_per_testcase(TestCase, Config) ->
54    ct_test_support:end_per_testcase(TestCase, Config).
55
56
57suite() ->
58    [{timetrap,{minutes,1}}].
59
60all() ->
61    all(suite).
62
63all(suite) ->
64      [
65       pre_init_per_suite,
66       post_init_per_suite,
67       pre_end_per_suite,
68       post_end_per_suite,
69       pre_init_per_group,
70       post_init_per_group,
71       pre_end_per_group,
72       post_end_per_group,
73       pre_init_per_testcase,
74       post_init_per_testcase,
75       pre_end_per_testcase,
76       post_end_per_testcase
77      ].
78
79
80%%--------------------------------------------------------------------
81%% TEST CASES
82%%--------------------------------------------------------------------
83
84%%%-----------------------------------------------------------------
85%%%
86pre_init_per_suite(Config) ->
87    CfgFile = gen_config(?FUNCTION_NAME,
88                         [{pre_init_per_suite,
89                           {curr_tc_SUITE,kill}}],
90                         Config),
91    ok = do_test(?FUNCTION_NAME,
92                 "curr_tc_SUITE.erl",
93                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
94                 Config).
95
96post_init_per_suite(Config) ->
97    CfgFile = gen_config(?FUNCTION_NAME,
98                         [{post_init_per_suite,
99                           {curr_tc_SUITE,kill}}],
100                         Config),
101    ok = do_test(?FUNCTION_NAME,
102                 "curr_tc_SUITE.erl",
103                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
104                 Config).
105
106pre_end_per_suite(Config) ->
107    CfgFile = gen_config(?FUNCTION_NAME,
108                         [{pre_end_per_suite,
109                           {curr_tc_SUITE,kill}}],
110                         Config),
111    ok = do_test(?FUNCTION_NAME,
112                 "curr_tc_SUITE.erl",
113                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
114                 Config).
115
116post_end_per_suite(Config) ->
117    CfgFile = gen_config(?FUNCTION_NAME,
118                         [{post_end_per_suite,
119                           {curr_tc_SUITE,kill}}],
120                         Config),
121    ok = do_test(?FUNCTION_NAME,
122                 "curr_tc_SUITE.erl",
123                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
124                 Config).
125
126
127pre_init_per_group(Config) ->
128    CfgFile = gen_config(?FUNCTION_NAME,
129                         [{pre_init_per_group,
130                           {curr_tc_SUITE,g,kill}}],
131                         Config),
132    ok = do_test(?FUNCTION_NAME,
133                 "curr_tc_SUITE.erl",
134                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
135                 Config).
136
137post_init_per_group(Config) ->
138    CfgFile = gen_config(?FUNCTION_NAME,
139                         [{post_init_per_group,
140                           {curr_tc_SUITE,g,kill}}],
141                         Config),
142    ok = do_test(?FUNCTION_NAME,
143                 "curr_tc_SUITE.erl",
144                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
145                 Config).
146
147pre_end_per_group(Config) ->
148    CfgFile = gen_config(?FUNCTION_NAME,
149                         [{pre_end_per_group,
150                           {curr_tc_SUITE,g,kill}}],
151                         Config),
152    ok = do_test(?FUNCTION_NAME,
153                 "curr_tc_SUITE.erl",
154                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
155                 Config).
156
157post_end_per_group(Config) ->
158    CfgFile = gen_config(?FUNCTION_NAME,
159                         [{post_end_per_group,
160                           {curr_tc_SUITE,g,kill}}],
161                         Config),
162    ok = do_test(?FUNCTION_NAME,
163                 "curr_tc_SUITE.erl",
164                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
165                 Config).
166
167pre_init_per_testcase(Config) ->
168    CfgFile = gen_config(?FUNCTION_NAME,
169                         [{pre_init_per_testcase,
170                           {curr_tc_SUITE,tc1,kill}}],
171                         Config),
172    ok = do_test(?FUNCTION_NAME,
173                 "curr_tc_SUITE.erl",
174                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
175                 Config).
176
177post_init_per_testcase(Config) ->
178    CfgFile = gen_config(?FUNCTION_NAME,
179                         [{post_init_per_testcase,
180                           {curr_tc_SUITE,tc1,{timeout,5000}}}],
181                         Config),
182    ok = do_test(?FUNCTION_NAME,
183                 "curr_tc_SUITE.erl",
184                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
185                 Config).
186
187pre_end_per_testcase(Config) ->
188    CfgFile = gen_config(?FUNCTION_NAME,
189                         [{pre_end_per_testcase,
190                           {curr_tc_SUITE,tc1,{timeout,5000}}}],
191                         Config),
192    ok = do_test(?FUNCTION_NAME,
193                 "curr_tc_SUITE.erl",
194                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
195                 Config).
196
197post_end_per_testcase(Config) ->
198    CfgFile = gen_config(?FUNCTION_NAME,
199                         [{post_end_per_testcase,
200                           {curr_tc_SUITE,tc1,kill}}],
201                         Config),
202    ok = do_test(?FUNCTION_NAME,
203                 "curr_tc_SUITE.erl",
204                 [{ct_hooks,[ct_util_cth]},{config,CfgFile}],
205                 Config).
206
207%%%-----------------------------------------------------------------
208%%% HELP FUNCTIONS
209%%%-----------------------------------------------------------------
210
211do_test(Tag, Suite, RunTestArgs, Config) ->
212    do_test(Tag, Suite, RunTestArgs, Config, 2).
213
214do_test(Tag, Suite0, RunTestArgs, Config, EC) ->
215    DataDir = ?config(data_dir, Config),
216    Suite = filename:join([DataDir,Suite0]),
217    {Opts,ERPid} = setup([{suite,Suite}]++[{label,Tag}|RunTestArgs],Config),
218    Res = ct_test_support:run(Opts, Config),
219    Events = ct_test_support:get_events(ERPid, Config),
220    %% io:format("~p~n",[Events]),
221
222    ct_test_support:log_events(Tag,
223			       reformat(Events, ?eh),
224			       ?config(priv_dir, Config),
225			       Opts),
226
227    TestEvents = events_to_check(Tag, EC),
228    ok = ct_test_support:verify_events(TestEvents, Events, Config),
229    Res.
230
231setup(Test, Config) ->
232    Opts0 = ct_test_support:get_opts(Config),
233    Level = ?config(trace_level, Config),
234    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
235    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
236    ERPid = ct_test_support:start_event_receiver(Config),
237    {Opts,ERPid}.
238
239reformat(Events, EH) ->
240    ct_test_support:reformat(Events, EH).
241
242gen_config(Name,KeyVals,Config) ->
243    PrivDir = ?config(priv_dir,Config),
244    File = filename:join(PrivDir,atom_to_list(Name)++".cfg"),
245    ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}])
246                               || {Key,Value} <- KeyVals]),
247    File.
248
249%%%-----------------------------------------------------------------
250%%% TEST EVENTS
251%%%-----------------------------------------------------------------
252events_to_check(Test) ->
253    %% 2 tests (ct:run_test + script_start) is default
254    events_to_check(Test, 2).
255
256events_to_check(_, 0) ->
257    [];
258events_to_check(Test, N) ->
259    test_events(Test) ++ events_to_check(Test, N-1).
260
261test_events(IPS) when IPS=:=pre_init_per_suite; IPS=:=post_init_per_suite ->
262    S = curr_tc_SUITE,
263    FwFunc =
264        case IPS of
265            pre_init_per_suite -> init_tc;
266            post_init_per_suite -> end_tc
267        end,
268    E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}},
269    [
270     {?eh,start_logging,{'DEF','RUNDIR'}},
271     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
272     {?eh,start_info,{1,1,4}},
273     {?eh,tc_start,{S,init_per_suite}},
274     {?eh,tc_done,{S,init_per_suite,E}},
275     {?eh,tc_auto_skip,{S,tc1,{failed,{S,init_per_suite,E}}}},
276     {?eh,tc_auto_skip,{S,tc2,{failed,{S,init_per_suite,E}}}},
277     {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_suite,E}}}},
278     {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_suite,E}}}},
279     {?eh,test_stats,{0,0,{0,4}}},
280     {?eh,tc_auto_skip,{S,end_per_suite,{failed,{S,init_per_suite,E}}}},
281     {?eh,test_done,{'DEF','STOP_TIME'}},
282     {?eh,stop_logging,[]}
283    ];
284
285test_events(EPS) when EPS=:=pre_end_per_suite; EPS=:=post_end_per_suite ->
286    S = curr_tc_SUITE,
287    FwFunc =
288        case EPS of
289            pre_end_per_suite -> init_tc;
290            post_end_per_suite -> end_tc
291        end,
292    E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}},
293    [
294     {?eh,start_logging,{'DEF','RUNDIR'}},
295     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
296     {?eh,start_info,{1,1,4}},
297     {?eh,tc_start,{S,init_per_suite}},
298     {?eh,tc_done,{S,init_per_suite,ok}},
299     {?eh,tc_start,{S,tc1}},
300     {?eh,tc_done,{S,tc1,ok}},
301     {?eh,test_stats,{1,0,{0,0}}},
302     {?eh,tc_start,{S,tc2}},
303     {?eh,tc_done,{S,tc2,ok}},
304     {?eh,test_stats,{2,0,{0,0}}},
305     [{?eh,tc_start,{S,{init_per_group,g,[]}}},
306      {?eh,tc_done,{S,{init_per_group,g,[]},ok}},
307      {?eh,tc_start,{S,tc1}},
308      {?eh,tc_done,{S,tc1,ok}},
309      {?eh,test_stats,{3,0,{0,0}}},
310      {?eh,tc_start,{S,tc2}},
311      {?eh,tc_done,{S,tc2,ok}},
312      {?eh,test_stats,{4,0,{0,0}}},
313      {?eh,tc_start,{S,{end_per_group,g,[]}}},
314      {?eh,tc_done,{S,{end_per_group,g,[]},ok}}],
315     {?eh,tc_start,{S,end_per_suite}},
316     {?eh,tc_done,{S,end_per_suite,E}},
317     {?eh,test_done,{'DEF','STOP_TIME'}},
318     {?eh,stop_logging,[]}
319    ];
320
321test_events(IPG) when IPG=:=pre_init_per_group; IPG=:=post_init_per_group ->
322    S = curr_tc_SUITE,
323    FwFunc =
324        case IPG of
325            pre_init_per_group -> init_tc;
326            post_init_per_group -> end_tc
327        end,
328    E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}},
329    [
330     {?eh,start_logging,{'DEF','RUNDIR'}},
331     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
332     {?eh,start_info,{1,1,4}},
333     {?eh,tc_start,{S,init_per_suite}},
334     {?eh,tc_done,{S,init_per_suite,ok}},
335     {?eh,tc_start,{S,tc1}},
336     {?eh,tc_done,{S,tc1,ok}},
337     {?eh,test_stats,{1,0,{0,0}}},
338     {?eh,tc_start,{S,tc2}},
339     {?eh,tc_done,{S,tc2,ok}},
340     {?eh,test_stats,{2,0,{0,0}}},
341     [{?eh,tc_start,{S,{init_per_group,g,[]}}},
342      {?eh,tc_done,{S,{init_per_group,g,[]},E}},
343      {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_group,E}}}},
344      {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_group,E}}}},
345      {?eh,test_stats,{2,0,{0,2}}},
346      {?eh,tc_auto_skip,{S,{end_per_group,g},{failed,{S,init_per_group,E}}}}],
347     {?eh,tc_start,{S,end_per_suite}},
348     {?eh,tc_done,{S,end_per_suite,ok}},
349     {?eh,test_done,{'DEF','STOP_TIME'}},
350     {?eh,stop_logging,[]}
351    ];
352
353test_events(EPG) when EPG=:=pre_end_per_group; EPG=:=post_end_per_group ->
354    S = curr_tc_SUITE,
355    FwFunc =
356        case EPG of
357            pre_end_per_group -> init_tc;
358            post_end_per_group -> end_tc
359        end,
360    E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}},
361    [
362     {?eh,start_logging,{'DEF','RUNDIR'}},
363     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
364     {?eh,start_info,{1,1,4}},
365     {?eh,tc_start,{S,init_per_suite}},
366     {?eh,tc_done,{S,init_per_suite,ok}},
367     {?eh,tc_start,{S,tc1}},
368     {?eh,tc_done,{S,tc1,ok}},
369     {?eh,test_stats,{1,0,{0,0}}},
370     {?eh,tc_start,{S,tc2}},
371     {?eh,tc_done,{S,tc2,ok}},
372     {?eh,test_stats,{2,0,{0,0}}},
373     [{?eh,tc_start,{S,{init_per_group,g,[]}}},
374      {?eh,tc_done,{S,{init_per_group,g,[]},ok}},
375      {?eh,tc_start,{S,tc1}},
376      {?eh,tc_done,{S,tc1,ok}},
377      {?eh,test_stats,{3,0,{0,0}}},
378      {?eh,tc_start,{S,tc2}},
379      {?eh,tc_done,{S,tc2,ok}},
380      {?eh,test_stats,{4,0,{0,0}}},
381      {?eh,tc_start,{S,{end_per_group,g,[]}}},
382      {?eh,tc_done,{S,{end_per_group,g,[]},E}}],
383     {?eh,tc_start,{S,end_per_suite}},
384     {?eh,tc_done,{S,end_per_suite,ok}},
385     {?eh,test_done,{'DEF','STOP_TIME'}},
386     {?eh,stop_logging,[]}
387    ];
388
389test_events(IPTC) when IPTC=:=pre_init_per_testcase;
390                       IPTC=:=post_init_per_testcase ->
391    S = curr_tc_SUITE,
392    E = case IPTC of
393            pre_init_per_testcase ->
394                {failed,{ct_framework,init_tc,{test_case_failed,hahahahahah}}};
395            post_init_per_testcase ->
396                {failed,{ct_framework,end_tc,{timetrap,3000}}}
397        end,
398    [
399     {?eh,start_logging,{'DEF','RUNDIR'}},
400     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
401     {?eh,start_info,{1,1,4}},
402     {?eh,tc_start,{S,init_per_suite}},
403     {?eh,tc_done,{S,init_per_suite,ok}},
404     {?eh,tc_start,{S,tc1}},
405     {?eh,tc_done,{S,tc1,{auto_skipped,E}}},
406     {?eh,test_stats,{0,0,{0,1}}},
407     {?eh,tc_start,{S,tc2}},
408     {?eh,tc_done,{S,tc2,ok}},
409     {?eh,test_stats,{1,0,{0,1}}},
410     [{?eh,tc_start,{S,{init_per_group,g,[]}}},
411      {?eh,tc_done,{S,{init_per_group,g,[]},ok}},
412      {?eh,tc_start,{S,tc1}},
413      {?eh,tc_done,{S,tc1,{auto_skipped,E}}},
414      {?eh,test_stats,{1,0,{0,2}}},
415      {?eh,tc_start,{S,tc2}},
416      {?eh,tc_done,{S,tc2,ok}},
417      {?eh,test_stats,{2,0,{0,2}}},
418      {?eh,tc_start,{S,{end_per_group,g,[]}}},
419      {?eh,tc_done,{S,{end_per_group,g,[]},ok}}],
420     {?eh,tc_start,{S,end_per_suite}},
421     {?eh,tc_done,{S,end_per_suite,ok}},
422     {?eh,test_done,{'DEF','STOP_TIME'}},
423     {?eh,stop_logging,[]}
424    ];
425
426test_events(EPTC) when EPTC=:=pre_end_per_testcase; EPTC=:=post_end_per_testcase->
427    S = curr_tc_SUITE,
428    [
429     {?eh,start_logging,{'DEF','RUNDIR'}},
430     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
431     {?eh,start_info,{1,1,4}},
432     {?eh,tc_start,{S,tc1}},
433     {?eh,tc_done,{S,tc1,ok}},
434     {?eh,test_stats,{1,0,{0,0}}},
435     {?eh,tc_start,{S,tc2}},
436     {?eh,tc_done,{S,tc2,ok}},
437     {?eh,test_stats,{2,0,{0,0}}},
438     [{?eh,tc_start,{S,{init_per_group,g,[]}}},
439      {?eh,tc_done,{S,{init_per_group,g,[]},ok}},
440      {?eh,tc_start,{S,tc1}},
441      {?eh,tc_done,{S,tc1,ok}},
442      {?eh,test_stats,{3,0,{0,0}}},
443      {?eh,tc_start,{S,tc2}},
444      {?eh,tc_done,{S,tc2,ok}},
445      {?eh,test_stats,{4,0,{0,0}}},
446      {?eh,tc_start,{S,{end_per_group,g,[]}}},
447      {?eh,tc_done,{S,{end_per_group,g,[]},ok}}],
448     {?eh,tc_start,{S,end_per_suite}},
449     {?eh,tc_done,{S,end_per_suite,ok}},
450     {?eh,test_done,{'DEF','STOP_TIME'}},
451     {?eh,stop_logging,[]}
452    ].
453
454%% test events help functions
455contains(List) ->
456    fun(Proplist) when is_list(Proplist) ->
457	    contains(List,Proplist)
458    end.
459
460contains([{not_in_order,List}|T],Rest) ->
461    contains_parallel(List,Rest),
462    contains(T,Rest);
463contains([{Ele,Pos}|T] = L,[H|T2]) ->
464    case element(Pos,H) of
465	Ele ->
466	    contains(T,T2);
467	_ ->
468	    contains(L,T2)
469    end;
470contains([Ele|T],[{Ele,_}|T2])->
471    contains(T,T2);
472contains([Ele|T],[Ele|T2])->
473    contains(T,T2);
474contains(List,[_|T]) ->
475    contains(List,T);
476contains([],_) ->
477    match.
478
479contains_parallel([Key | T], Elems) ->
480    contains([Key],Elems),
481    contains_parallel(T,Elems);
482contains_parallel([],_Elems) ->
483    match.
484
485not_contains(List) ->
486    fun(Proplist) when is_list(Proplist) ->
487	    [] = [Ele || {Ele,_} <- Proplist,
488			 Test <- List,
489			 Test =:= Ele]
490    end.
491