1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2009-2020. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20-module(ct_tc_repeat_SUITE). 21 22-compile(export_all). 23 24-include_lib("common_test/include/ct.hrl"). 25-include_lib("common_test/include/ct_event.hrl"). 26 27-define(eh, ct_test_support_eh). 28 29%%-------------------------------------------------------------------- 30%% TEST SERVER CALLBACK FUNCTIONS 31%%-------------------------------------------------------------------- 32 33%%-------------------------------------------------------------------- 34%% Description: Since Common Test starts another Test Server 35%% instance, the tests need to be performed on a separate node (or 36%% there will be clashes with logging processes etc). 37%%-------------------------------------------------------------------- 38init_per_suite(Config) -> 39 DataDir = ?config(data_dir, Config), 40 ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). 41 42end_per_suite(Config) -> 43 ct_test_support:end_per_suite(Config). 44 45init_per_testcase(TestCase, Config) -> 46 ct_test_support:init_per_testcase(TestCase, Config). 47 48end_per_testcase(TestCase, Config) -> 49 ct_test_support:end_per_testcase(TestCase, Config). 50 51 52suite() -> 53 [{timetrap,{minutes,1}}]. 54 55all() -> 56 all(suite). 57 58all(suite) -> 59 [ 60 repeat, 61 repeat_parallel_until_ok, 62 repeat_parallel_until_fail, 63 repeat_sequence_until_ok, 64 repeat_sequence_until_fail, 65 pick_one_test_from_group, 66 pick_one_test_from_subgroup 67 ]. 68 69 70%%-------------------------------------------------------------------- 71%% TEST CASES 72%%-------------------------------------------------------------------- 73 74%%%----------------------------------------------------------------- 75%%% 76%% Test post_groups and post_all hook callbacks, introduced by OTP-14746 77repeat(Config) -> 78 ok = do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [], [], Config). 79 80repeat_parallel_until_ok(Config) -> 81 {error,{{illegal_combination,{parallel,repeat_until_ok}},_}} = 82 do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_ok}], 83 [], Config, 1, []). 84 85repeat_parallel_until_fail(Config) -> 86 {error,{{illegal_combination,{parallel,repeat_until_fail}},_}} = 87 do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_fail}], 88 [], Config, 1, []). 89 90repeat_sequence_until_ok(Config) -> 91 {error,{{illegal_combination,{sequence,repeat_until_ok}},_}} = 92 do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_ok}], 93 [], Config, 1, []). 94 95repeat_sequence_until_fail(Config) -> 96 {error,{{illegal_combination,{sequence,repeat_until_fail}},_}} = 97 do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_fail}], 98 [], Config, 1, []). 99 100pick_one_test_from_group(Config) -> 101 do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_mixed},{testcase,tc2}], 102 [], Config, 1, []). 103 104pick_one_test_from_subgroup(Config) -> 105 do_test(?FUNCTION_NAME, "tc_repeat_SUITE", 106 [{group,[[g_mixed,subgroup]]},{testcase,tc2}], 107 [], Config, 1, []). 108 109 110%%%----------------------------------------------------------------- 111%%% HELP FUNCTIONS 112%%%----------------------------------------------------------------- 113 114do_test(Tag, Suite, WTT, CTHs, Config) -> 115 do_test(Tag, Suite, WTT, CTHs, Config, 2, []). 116 117do_test(Tag, Suite0, WTT, CTHs, Config, EC, ExtraOpts) -> 118 DataDir = ?config(data_dir, Config), 119 Suite = filename:join([DataDir,Suite0]), 120 {Opts,ERPid} = 121 setup([{suite,Suite}|WTT]++[{ct_hooks,CTHs},{label,Tag}|ExtraOpts], 122 Config), 123 Res = ct_test_support:run(Opts, Config), 124 Events = ct_test_support:get_events(ERPid, Config), 125 %% io:format("~p~n",[Events]), 126 127 ct_test_support:log_events(Tag, 128 reformat(Events, ?eh), 129 ?config(priv_dir, Config), 130 Opts), 131 132 TestEvents = events_to_check(Tag, EC), 133 ok = ct_test_support:verify_events(TestEvents, Events, Config), 134 Res. 135 136setup(Test, Config) -> 137 Opts0 = ct_test_support:get_opts(Config), 138 Level = ?config(trace_level, Config), 139 EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], 140 Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], 141 ERPid = ct_test_support:start_event_receiver(Config), 142 {Opts,ERPid}. 143 144reformat(Events, EH) -> 145 ct_test_support:reformat(Events, EH). 146 147gen_config(Name,KeyVals,Config) -> 148 PrivDir = ?config(priv_dir,Config), 149 File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), 150 ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) 151 || {Key,Value} <- KeyVals]), 152 File. 153 154%%%----------------------------------------------------------------- 155%%% TEST EVENTS 156%%%----------------------------------------------------------------- 157events_to_check(Test) -> 158 %% 2 tests (ct:run_test + script_start) is default 159 events_to_check(Test, 2). 160 161events_to_check(_, 0) -> 162 []; 163events_to_check(Test, N) -> 164 test_events(Test) ++ events_to_check(Test, N-1). 165 166test_events(repeat) -> 167 S = tc_repeat_SUITE, 168 [ 169 {?eh,start_logging,{'DEF','RUNDIR'}}, 170 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 171 {?eh,start_info,{1,1,unknown}}, 172 173 %% tc1, {repeat,2} 174 {?eh,tc_start,{S,tc1}}, 175 {?eh,tc_done,{S,tc1,ok}}, 176 {?eh,test_stats,{1,0,{0,0}}}, 177 {?eh,tc_start,{S,tc1}}, 178 {?eh,tc_done,{S,tc1,ok}}, 179 {?eh,test_stats,{2,0,{0,0}}}, 180 %% tc2, {repeat_until_ok,3} 181 {?eh,tc_start,{S,tc2}}, 182 {?eh,tc_done,{S,tc2,ok}}, 183 {?eh,test_stats,{3,0,{0,0}}}, 184 %% tc3, {repeat_until_ok,3} 185 {?eh,tc_start,{S,tc3}}, 186 {?eh,tc_done,{tc_repeat_SUITE,tc3, 187 {failed,{error,{test_case_failed,always_fail}}}}}, 188 {?eh,test_stats,{3,1,{0,0}}}, 189 {?eh,tc_start,{S,tc3}}, 190 {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}}, 191 {?eh,test_stats,{3,2,{0,0}}}, 192 {?eh,tc_start,{S,tc3}}, 193 {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}}, 194 {?eh,test_stats,{3,3,{0,0}}}, 195 %% tc4, {repeat_until_fail,3} 196 {?eh,tc_start,{S,tc4}}, 197 {?eh,tc_done,{S,tc4,ok}}, 198 {?eh,test_stats,{4,3,{0,0}}}, 199 {?eh,tc_start,{S,tc4}}, 200 {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, 201 {?eh,test_stats,{4,4,{0,0}}}, 202 %% g, tc1, {repeat,2} 203 {?eh,tc_start,{S,{init_per_group,g,[]}}}, 204 {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, 205 {?eh,tc_start,{S,tc1}}, 206 {?eh,tc_done,{S,tc1,ok}}, 207 {?eh,test_stats,{5,4,{0,0}}}, 208 {?eh,tc_start,{S,tc1}}, 209 {?eh,tc_done,{S,tc1,ok}}, 210 {?eh,test_stats,{6,4,{0,0}}}, 211 {?eh,tc_start,{S,{end_per_group,g,[]}}}, 212 {?eh,tc_done,{S,{end_per_group,g,[]},ok}}, 213 %% g_until_ok, tc2, {repeat_until_ok,3} 214 {?eh,tc_start,{S,{init_per_group,g_until_ok,[]}}}, 215 {?eh,tc_done,{S,{init_per_group,g_until_ok,[]},ok}}, 216 {?eh,tc_start,{S,tc2}}, 217 {?eh,tc_done,{S,tc2,ok}}, 218 {?eh,test_stats,{7,4,{0,0}}}, 219 {?eh,tc_start,{S,{end_per_group,g_until_ok,[]}}}, 220 {?eh,tc_done,{S,{end_per_group,g_until_ok,[]},ok}}, 221 %% g_until_fail, tc4, {repeat_until_fail,3} 222 {?eh,tc_start,{S,{init_per_group,g_until_fail,[]}}}, 223 {?eh,tc_done,{S,{init_per_group,g_until_fail,[]},ok}}, 224 {?eh,tc_start,{S,tc4}}, 225 {?eh,tc_done,{S,tc4,ok}}, 226 {?eh,test_stats,{8,4,{0,0}}}, 227 {?eh,tc_start,{S,tc4}}, 228 {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, 229 {?eh,test_stats,{8,5,{0,0}}}, 230 {?eh,tc_start,{S,{end_per_group,g_until_fail,[]}}}, 231 {?eh,tc_done,{S,{end_per_group,g_until_fail,[]},ok}}, 232 %% g, parallel, tc1, {repeat,2} 233 {parallel, 234 [{?eh,tc_start,{S,{init_per_group,g,[parallel]}}}, 235 {?eh,tc_done,{S,{init_per_group,g,[parallel]},ok}}, 236 {?eh,tc_start,{S,tc1}}, 237 {?eh,tc_done,{S,tc1,ok}}, 238 {?eh,test_stats,{9,5,{0,0}}}, 239 {?eh,tc_start,{S,tc1}}, 240 {?eh,tc_done,{S,tc1,ok}}, 241 {?eh,test_stats,{10,5,{0,0}}}, 242 {?eh,tc_start,{S,{end_per_group,g,[parallel]}}}, 243 {?eh,tc_done,{S,{end_per_group,g,[parallel]},ok}}]}, 244 %% g, sequence, tc1, {repeat,2} 245 {?eh,tc_start,{S,{init_per_group,g,[sequence]}}}, 246 {?eh,tc_done,{S,{init_per_group,g,[sequence]},ok}}, 247 {?eh,tc_start,{S,tc1}}, 248 {?eh,tc_done,{S,tc1,ok}}, 249 {?eh,test_stats,{11,5,{0,0}}}, 250 {?eh,tc_start,{S,tc1}}, 251 {?eh,tc_done,{S,tc1,ok}}, 252 {?eh,test_stats,{12,5,{0,0}}}, 253 {?eh,tc_start,{S,{end_per_group,g,[sequence]}}}, 254 {?eh,tc_done,{S,{end_per_group,g,[sequence]},ok}}, 255 %% g_sequence_skip_rest, 256 {?eh,tc_start,{S,{init_per_group,g_mixed,[sequence]}}}, 257 {?eh,tc_done,{S,{init_per_group,g_mixed,[sequence]},ok}}, 258 {?eh,tc_start,{S,tc1}}, 259 {?eh,tc_done,{S,tc1,ok}}, 260 {?eh,test_stats,{13,5,{0,0}}}, 261 {?eh,tc_start,{S,tc1}}, 262 {?eh,tc_done,{S,tc1,ok}}, 263 {?eh,test_stats,{14,5,{0,0}}}, 264 {?eh,tc_start,{S,tc4}}, 265 {?eh,tc_done,{S,tc4,ok}}, 266 {?eh,test_stats,{15,5,{0,0}}}, 267 {?eh,tc_start,{S,tc4}}, 268 {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, 269 {?eh,test_stats,{15,6,{0,0}}}, 270 %% ----> fail in sequence, so skip rest 271 {?eh,tc_auto_skip,{S,{tc4,g_mixed}, % last of current repeat tc4 272 {failed,{tc_repeat_SUITE,tc4}}}}, 273 {?eh,test_stats,{15,6,{0,1}}}, 274 {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1 275 {failed,{tc_repeat_SUITE,tc4}}}}, 276 {?eh,test_stats,{15,6,{0,2}}}, 277 {?eh,tc_auto_skip,{S,{tc1,g}, % group g, tc1, {repeat,2} 278 {failed,{tc_repeat_SUITE,tc4}}}}, 279 {?eh,test_stats,{15,6,{0,3}}}, 280 {?eh,tc_auto_skip,{S,{tc1,subgroup}, % subgroup, single tc1 281 {failed,{tc_repeat_SUITE,tc4}}}}, 282 {?eh,test_stats,{15,6,{0,4}}}, 283 {?eh,tc_auto_skip,{S,{tc2,subgroup}, % subgroup, tc2, {repeat,2} 284 {failed,{tc_repeat_SUITE,tc4}}}}, 285 {?eh,test_stats,{15,6,{0,5}}}, 286 {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % tc2, {repeat,2} 287 {failed,{tc_repeat_SUITE,tc4}}}}, 288 {?eh,test_stats,{15,6,{0,6}}}, 289 {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % single tc2 290 {failed,{tc_repeat_SUITE,tc4}}}}, 291 {?eh,test_stats,{15,6,{0,7}}}, 292 {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % tc1, {repeat,2} 293 {failed,{tc_repeat_SUITE,tc4}}}}, 294 {?eh,test_stats,{15,6,{0,8}}}, 295 {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1 296 {failed,{tc_repeat_SUITE,tc4}}}}, 297 {?eh,test_stats,{15,6,{0,9}}}, 298 {?eh,tc_start,{S,{end_per_group,g_mixed,'_'}}}, 299 {?eh,tc_done,{S,{end_per_group,g_mixed,'_'},ok}}, 300 %% done 301 {?eh,test_done,{'DEF','STOP_TIME'}}, 302 {?eh,stop_logging,[]} 303 ]; 304 305test_events(repeat_parallel_until_ok) -> 306 [ 307 {?eh,start_logging,{'DEF','RUNDIR'}}, 308 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 309 {?eh,tc_start,{'_',{init_per_group,g_parallel_until_ok,[parallel]}}}, 310 {?eh,tc_done,{'_',{init_per_group,g_parallel_until_ok,[parallel]},ok}}, 311 {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_ok}},'_'}}, 312 {?eh,test_done,{'DEF','STOP_TIME'}}, 313 {?eh,stop_logging,[]} 314 ]; 315 316test_events(repeat_parallel_until_fail) -> 317 [ 318 {?eh,start_logging,{'DEF','RUNDIR'}}, 319 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 320 {?eh,tc_start,{'_',{init_per_group,g_parallel_until_fail,[parallel]}}}, 321 {?eh,tc_done,{'_',{init_per_group,g_parallel_until_fail,[parallel]},ok}}, 322 {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_fail}},'_'}}, 323 {?eh,test_done,{'DEF','STOP_TIME'}}, 324 {?eh,stop_logging,[]} 325 ]; 326 327test_events(repeat_sequence_until_ok) -> 328 [ 329 {?eh,start_logging,{'DEF','RUNDIR'}}, 330 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 331 {?eh,tc_start,{'_',{init_per_group,g_sequence_until_ok,[sequence]}}}, 332 {?eh,tc_done,{'_',{init_per_group,g_sequence_until_ok,[sequence]},ok}}, 333 {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_ok}},'_'}}, 334 {?eh,test_done,{'DEF','STOP_TIME'}}, 335 {?eh,stop_logging,[]} 336 ]; 337 338test_events(repeat_sequence_until_fail) -> 339 [ 340 {?eh,start_logging,{'DEF','RUNDIR'}}, 341 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 342 {?eh,tc_start,{'_',{init_per_group,g_sequence_until_fail,[sequence]}}}, 343 {?eh,tc_done,{'_',{init_per_group,g_sequence_until_fail,[sequence]},ok}}, 344 {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_fail}},'_'}}, 345 {?eh,test_done,{'DEF','STOP_TIME'}}, 346 {?eh,stop_logging,[]} 347 ]; 348 349test_events(pick_one_test_from_group) -> 350 [ 351 {?eh,start_logging,{'DEF','RUNDIR'}}, 352 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 353 {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}}, 354 {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}}, 355 {negative, 356 {?eh,tc_start,{'_',tc1}}, 357 {?eh,tc_start,{'_',tc2}}}, % single tc2 358 {?eh,tc_done,{'_',tc2,ok}}, 359 {?eh,tc_start,{'_',tc2}}, % tc2, {repeat,2} 360 {?eh,tc_done,{'_',tc2,ok}}, 361 {?eh,tc_start,{'_',tc2}}, 362 {?eh,tc_done,{'_',tc2,ok}}, 363 {negative, 364 {?eh,tc_start,{'_',tc1}}, 365 {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}}, 366 {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}}, 367 {?eh,test_done,{'DEF','STOP_TIME'}}, 368 {?eh,stop_logging,[]} 369 ]; 370 371test_events(pick_one_test_from_subgroup) -> 372 [ 373 {?eh,start_logging,{'DEF','RUNDIR'}}, 374 {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, 375 {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}}, 376 {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}}, 377 {negative, 378 {?eh,tc_start,{'_',tc2}}, 379 {?eh,tc_start,{'_',{init_per_group,subgroup,[]}}}}, 380 {?eh,tc_done,{'_',{init_per_group,subgroup,[]},ok}}, 381 {negative, 382 {?eh,tc_start,{'_',tc1}}, 383 {?eh,tc_start,{'_',tc2}}}, % tc2, {repeat,2} 384 {?eh,tc_done,{'_',tc2,ok}}, 385 {?eh,tc_start,{'_',tc2}}, 386 {?eh,tc_done,{'_',tc2,ok}}, 387 {negative, 388 {?eh,tc_start,{'_',tc1}}, 389 {?eh,tc_start,{'_',{end_per_group,subgroup,[]}}}}, 390 {?eh,tc_done,{'_',{end_per_group,subgroup,[]},ok}}, 391 {negative, 392 {?eh,tc_start,{'_',tc2}}, 393 {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}}, 394 {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}}, 395 {?eh,test_done,{'DEF','STOP_TIME'}}, 396 {?eh,stop_logging,[]} 397 ]; 398 399test_events(ok) -> 400 ok. 401 402%% test events help functions 403contains(List) -> 404 fun(Proplist) when is_list(Proplist) -> 405 contains(List,Proplist) 406 end. 407 408contains([{not_in_order,List}|T],Rest) -> 409 contains_parallel(List,Rest), 410 contains(T,Rest); 411contains([{Ele,Pos}|T] = L,[H|T2]) -> 412 case element(Pos,H) of 413 Ele -> 414 contains(T,T2); 415 _ -> 416 contains(L,T2) 417 end; 418contains([Ele|T],[{Ele,_}|T2])-> 419 contains(T,T2); 420contains([Ele|T],[Ele|T2])-> 421 contains(T,T2); 422contains(List,[_|T]) -> 423 contains(List,T); 424contains([],_) -> 425 match. 426 427contains_parallel([Key | T], Elems) -> 428 contains([Key],Elems), 429 contains_parallel(T,Elems); 430contains_parallel([],_Elems) -> 431 match. 432 433not_contains(List) -> 434 fun(Proplist) when is_list(Proplist) -> 435 [] = [Ele || {Ele,_} <- Proplist, 436 Test <- List, 437 Test =:= Ele] 438 end. 439