1# Copyright 2011 Google Inc. 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above copyright 11# notice, this list of conditions and the following disclaimer in the 12# documentation and/or other materials provided with the distribution. 13# * Neither the name of Google Inc. nor the names of its contributors 14# may be used to endorse or promote products derived from this software 15# without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 30utils_test_case no_args 31no_args_body() { 32 cat >Kyuafile <<EOF 33syntax(2) 34test_suite("integration") 35atf_test_program{name="simple_all_pass"} 36EOF 37 utils_cp_helper simple_all_pass . 38 39 cat >experr <<EOF 40Usage error for command debug: Not enough arguments. 41Type 'kyua help debug' for usage information. 42EOF 43 atf_check -s exit:3 -o empty -e file:experr kyua debug 44} 45 46 47utils_test_case many_args 48many_args_body() { 49 cat >Kyuafile <<EOF 50syntax(2) 51test_suite("integration") 52atf_test_program{name="first"} 53atf_test_program{name="second"} 54EOF 55 utils_cp_helper simple_all_pass first 56 utils_cp_helper simple_all_pass second 57 58 cat >experr <<EOF 59Usage error for command debug: Too many arguments. 60Type 'kyua help debug' for usage information. 61EOF 62 atf_check -s exit:3 -o empty -e file:experr kyua debug first:pass \ 63 second:pass 64} 65 66 67utils_test_case one_arg__ok_pass 68one_arg__ok_pass_body() { 69 cat >Kyuafile <<EOF 70syntax(2) 71test_suite("integration") 72atf_test_program{name="first"} 73atf_test_program{name="second"} 74EOF 75 utils_cp_helper expect_all_pass first 76 utils_cp_helper simple_all_pass second 77 78 cat >expout <<EOF 79This is the stdout of pass 80second:pass -> passed 81EOF 82cat >experr <<EOF 83This is the stderr of pass 84EOF 85 atf_check -s exit:0 -o file:expout -e file:experr kyua debug second:pass 86} 87 88 89utils_test_case one_arg__ok_fail 90one_arg__ok_fail_body() { 91 cat >Kyuafile <<EOF 92syntax(2) 93test_suite("integration") 94atf_test_program{name="first"} 95EOF 96 utils_cp_helper simple_some_fail first 97 98 cat >expout <<EOF 99This is the stdout of fail 100first:fail -> failed: This fails on purpose 101EOF 102 cat >experr <<EOF 103This is the stderr of fail 104EOF 105 atf_check -s exit:1 -o file:expout -e file:experr kyua debug first:fail 106} 107 108 109utils_test_case one_arg__no_match 110one_arg__no_match_body() { 111 cat >Kyuafile <<EOF 112syntax(2) 113test_suite("integration") 114atf_test_program{name="first"} 115atf_test_program{name="second"} 116EOF 117 utils_cp_helper expect_all_pass first 118 utils_cp_helper simple_all_pass second 119 120 cat >experr <<EOF 121kyua: E: Unknown test case 'second:die'. 122EOF 123 atf_check -s exit:2 -o empty -e file:experr kyua debug second:die 124} 125 126 127utils_test_case one_arg__no_test_case 128one_arg__no_test_case_body() { 129 cat >experr <<EOF 130Usage error for command debug: 'foo' is not a test case identifier (missing ':'?). 131Type 'kyua help debug' for usage information. 132EOF 133 atf_check -s exit:3 -o empty -e file:experr kyua debug foo 134} 135 136 137utils_test_case one_arg__bad_filter 138one_arg__bad_filter_body() { 139 cat >experr <<EOF 140kyua: E: Test case component in 'foo:' is empty. 141EOF 142 atf_check -s exit:2 -o empty -e file:experr kyua debug foo: 143} 144 145 146utils_test_case body_and_cleanup 147body_and_cleanup_body() { 148 cat >Kyuafile <<EOF 149syntax(2) 150test_suite("integration") 151atf_test_program{name="single"} 152EOF 153 utils_cp_helper metadata single 154 155 cat >expout <<EOF 156single:with_cleanup -> passed 157EOF 158 atf_check -s exit:0 -o file:expout -e empty kyua debug \ 159 --stdout=saved.out --stderr=saved.err single:with_cleanup 160 161 cat >expout <<EOF 162Body message to stdout 163Cleanup message to stdout 164EOF 165 atf_check -s exit:0 -o file:expout -e empty cat saved.out 166 167 cat >experr <<EOF 168Body message to stderr 169Cleanup message to stderr 170EOF 171 atf_check -s exit:0 -o file:experr -e empty cat saved.err 172} 173 174 175utils_test_case stdout_stderr_flags 176stdout_stderr_flags_body() { 177 cat >Kyuafile <<EOF 178syntax(2) 179test_suite("integration") 180atf_test_program{name="first"} 181atf_test_program{name="second"} 182EOF 183 utils_cp_helper expect_all_pass first 184 utils_cp_helper simple_all_pass second 185 186 cat >expout <<EOF 187second:pass -> passed 188EOF 189 atf_check -s exit:0 -o file:expout -e empty kyua debug \ 190 --stdout=saved.out --stderr=saved.err second:pass 191 192 cat >expout <<EOF 193This is the stdout of pass 194EOF 195 cmp -s saved.out expout || atf_fail "--stdout did not redirect the" \ 196 "standard output to the desired file" 197 198 cat >experr <<EOF 199This is the stderr of pass 200EOF 201 cmp -s saved.err experr || atf_fail "--stderr did not redirect the" \ 202 "standard error to the desired file" 203} 204 205 206utils_test_case args_are_relative 207args_are_relative_body() { 208 mkdir root 209 cat >root/Kyuafile <<EOF 210syntax(2) 211test_suite("integration") 212include("subdir/Kyuafile") 213atf_test_program{name="prog"} 214EOF 215 utils_cp_helper simple_all_pass root/prog 216 217 mkdir root/subdir 218 cat >root/subdir/Kyuafile <<EOF 219syntax(2) 220test_suite("integration") 221atf_test_program{name="prog"} 222EOF 223 utils_cp_helper simple_some_fail root/subdir/prog 224 225 cat >expout <<EOF 226This is the stdout of fail 227subdir/prog:fail -> failed: This fails on purpose 228EOF 229 cat >experr <<EOF 230This is the stderr of fail 231EOF 232 atf_check -s exit:1 -o file:expout -e file:experr kyua debug \ 233 -k "$(pwd)/root/Kyuafile" subdir/prog:fail 234} 235 236 237utils_test_case only_load_used_test_programs 238only_load_used_test_programs_body() { 239 cat >Kyuafile <<EOF 240syntax(2) 241test_suite("integration") 242atf_test_program{name="first"} 243atf_test_program{name="second"} 244EOF 245 utils_cp_helper simple_all_pass first 246 utils_cp_helper bad_test_program second 247 248 cat >expout <<EOF 249This is the stdout of pass 250first:pass -> passed 251EOF 252 cat >experr <<EOF 253This is the stderr of pass 254EOF 255 CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE 256 atf_check -s exit:0 -o file:expout -e file:experr kyua debug first:pass 257 if test -f "${CREATE_COOKIE}"; then 258 atf_fail "An unmatched test case has been executed, which harms" \ 259 "performance" 260 fi 261} 262 263 264utils_test_case config_behavior 265config_behavior_body() { 266 cat >"my-config" <<EOF 267syntax(2) 268test_suites.suite1["X-the-variable"] = "value1" 269test_suites.suite2["X-the-variable"] = "override me" 270EOF 271 272 cat >Kyuafile <<EOF 273syntax(2) 274atf_test_program{name="config1", test_suite="suite1"} 275atf_test_program{name="config2", test_suite="suite2"} 276atf_test_program{name="config3", test_suite="suite3"} 277EOF 278 utils_cp_helper config config1 279 utils_cp_helper config config2 280 utils_cp_helper config config3 281 282 atf_check -s exit:1 -o match:'failed' -e empty \ 283 kyua -c my-config -v test_suites.suite2.X-the-variable=value2 \ 284 debug config1:get_variable 285 atf_check -s exit:0 -o match:'passed' -e empty \ 286 kyua -c my-config -v test_suites.suite2.X-the-variable=value2 \ 287 debug config2:get_variable 288 atf_check -s exit:0 -o match:'skipped' -e empty \ 289 kyua -c my-config -v test_suites.suite2.X-the-variable=value2 \ 290 debug config3:get_variable 291} 292 293 294utils_test_case build_root_flag 295build_root_flag_body() { 296 cat >Kyuafile <<EOF 297syntax(2) 298test_suite("integration") 299atf_test_program{name="first"} 300atf_test_program{name="second"} 301EOF 302 mkdir build 303 utils_cp_helper expect_all_pass build/first 304 utils_cp_helper simple_all_pass build/second 305 306 cat >expout <<EOF 307This is the stdout of pass 308second:pass -> passed 309EOF 310cat >experr <<EOF 311This is the stderr of pass 312EOF 313 atf_check -s exit:0 -o file:expout -e file:experr \ 314 kyua debug --build-root=build second:pass 315} 316 317 318utils_test_case kyuafile_flag__ok 319kyuafile_flag__ok_body() { 320 cat >Kyuafile <<EOF 321This file is bogus but it is not loaded. 322EOF 323 324 cat >myfile <<EOF 325syntax(2) 326test_suite("hello-world") 327atf_test_program{name="sometest"} 328EOF 329 utils_cp_helper simple_all_pass sometest 330 331 atf_check -s exit:0 -o match:passed -e empty kyua test -k myfile sometest 332 atf_check -s exit:0 -o match:passed -e empty kyua test --kyuafile=myfile \ 333 sometest 334} 335 336 337utils_test_case missing_kyuafile 338missing_kyuafile_body() { 339 cat >experr <<EOF 340kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 341EOF 342 atf_check -s exit:2 -o empty -e file:experr kyua debug foo:bar 343} 344 345 346utils_test_case bogus_kyuafile 347bogus_kyuafile_body() { 348 cat >Kyuafile <<EOF 349Hello, world. 350EOF 351 atf_check -s exit:2 -o empty \ 352 -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list 353} 354 355 356utils_test_case bogus_test_program 357bogus_test_program_body() { 358 cat >Kyuafile <<EOF 359syntax(2) 360test_suite("integration") 361atf_test_program{name="crash_on_list"} 362atf_test_program{name="non_executable"} 363EOF 364 utils_cp_helper bad_test_program crash_on_list 365 echo 'I am not executable' >non_executable 366 367 cat >experr <<EOF 368kyua: E: Unknown test case 'crash_on_list:a'. 369EOF 370 atf_check -s exit:2 -o empty -e file:experr kyua debug crash_on_list:a 371 372 cat >experr <<EOF 373kyua: E: Unknown test case 'non_executable:a'. 374EOF 375 atf_check -s exit:2 -o empty -e file:experr kyua debug non_executable:a 376 377 cat >expout <<EOF 378crash_on_list:__test_cases_list__ -> broken: Tester did not exit cleanly: kyua-atf-tester: Invalid test cases list header 'This is not a valid test program!' 379EOF 380 atf_check -s exit:1 -o file:expout -e empty kyua debug crash_on_list:__test_cases_list__ 381 382 cat >expout <<EOF 383non_executable:__test_cases_list__ -> broken: Tester did not exit cleanly: kyua-atf-tester: execvp failed: Permission denied 384EOF 385 atf_check -s exit:1 -o file:expout -e empty kyua debug non_executable:__test_cases_list__ 386} 387 388 389atf_init_test_cases() { 390 atf_add_test_case no_args 391 atf_add_test_case many_args 392 atf_add_test_case one_arg__ok_pass 393 atf_add_test_case one_arg__ok_fail 394 atf_add_test_case one_arg__no_match 395 atf_add_test_case one_arg__no_test_case 396 atf_add_test_case one_arg__bad_filter 397 398 atf_add_test_case body_and_cleanup 399 400 atf_add_test_case stdout_stderr_flags 401 402 atf_add_test_case args_are_relative 403 404 atf_add_test_case only_load_used_test_programs 405 406 atf_add_test_case config_behavior 407 408 atf_add_test_case build_root_flag 409 atf_add_test_case kyuafile_flag__ok 410 atf_add_test_case missing_kyuafile 411 atf_add_test_case bogus_kyuafile 412 atf_add_test_case bogus_test_program 413} 414