1# 2# Automated Testing Framework (atf) 3# 4# Copyright (c) 2007 The NetBSD Foundation, Inc. 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28# 29 30create_helpers() 31{ 32 mkdir dir1 33 cp $(atf_get_srcdir)/pass_helper dir1/tp1 34 cp $(atf_get_srcdir)/fail_helper dir1/tp2 35 cp $(atf_get_srcdir)/pass_helper tp3 36 cp $(atf_get_srcdir)/fail_helper tp4 37 38 cat >tp5 <<EOF 39#! $(atf-config -t atf_shell) 40echo foo 41EOF 42 chmod +x tp5 43 44 cat >Atffile <<EOF 45Content-Type: application/X-atf-atffile; version="1" 46 47prop: test-suite = atf 48 49tp: dir1 50tp: tp3 51tp: tp4 52tp: tp5 53EOF 54 55 cat >dir1/Atffile <<EOF 56Content-Type: application/X-atf-atffile; version="1" 57 58prop: test-suite = atf 59 60tp: tp1 61tp: tp2 62EOF 63} 64 65run_helpers() 66{ 67 mkdir etc 68 cat >etc/atf-run.hooks <<EOF 69#! $(atf-config -t atf_shell) 70 71info_start_hook() 72{ 73 atf_tps_writer_info "startinfo" "A value" 74} 75 76info_end_hook() 77{ 78 atf_tps_writer_info "endinfo" "Another value" 79} 80EOF 81 echo "Using atf-run to run helpers" 82 ATF_CONFDIR=$(pwd)/etc atf-run >tps.out 2>/dev/null 83 rm -rf etc 84} 85 86atf_test_case default 87default_head() 88{ 89 atf_set "descr" "Checks that the default output uses the ticker" \ 90 "format" 91} 92default_body() 93{ 94 create_helpers 95 run_helpers 96 97 # Check that the default output uses the ticker format. 98 atf_check -s eq:0 -o match:'test cases' -o match:'Failed test cases' \ 99 -o match:'Summary for' -e empty -x 'atf-report <tps.out' 100} 101 102# XXX The test for all expect_ values should be intermixed with the other 103# tests. However, to do that, we need to migrate to using C helpers for 104# simplicity in raising signals... 105atf_test_case expect 106expect_body() 107{ 108 ln -s "$(atf_get_srcdir)/../atf-run/expect_helpers" . 109 cat >Atffile <<EOF 110Content-Type: application/X-atf-atffile; version="1" 111 112prop: test-suite = atf 113 114tp: expect_helpers 115EOF 116 run_helpers 117 118# NO_CHECK_STYLE_BEGIN 119 cat >expout <<EOF 120tc, #.#, expect_helpers, death_and_exit, expected_death, Exit case 121tc, #.#, expect_helpers, death_and_signal, expected_death, Signal case 122tc, #.#, expect_helpers, death_but_pass, failed, Test case was expected to terminate abruptly but it continued execution 123tc, #.#, expect_helpers, exit_any_and_exit, expected_exit, Call will exit 124tc, #.#, expect_helpers, exit_but_pass, failed, Test case was expected to exit cleanly but it continued execution 125tc, #.#, expect_helpers, exit_code_and_exit, expected_exit, Call will exit 126tc, #.#, expect_helpers, fail_and_fail_check, expected_failure, And fail again: 2 checks failed as expected; see output for more details 127tc, #.#, expect_helpers, fail_and_fail_requirement, expected_failure, Fail reason: The failure 128tc, #.#, expect_helpers, fail_but_pass, failed, Test case was expecting a failure but none were raised 129tc, #.#, expect_helpers, pass_and_pass, passed 130tc, #.#, expect_helpers, pass_but_fail_check, failed, 1 checks failed; see output for more details 131tc, #.#, expect_helpers, pass_but_fail_requirement, failed, Some reason 132tc, #.#, expect_helpers, signal_any_and_signal, expected_signal, Call will signal 133tc, #.#, expect_helpers, signal_but_pass, failed, Test case was expected to receive a termination signal but it continued execution 134tc, #.#, expect_helpers, signal_no_and_signal, expected_signal, Call will signal 135tc, #.#, expect_helpers, timeout_and_hang, expected_timeout, Will overrun 136tc, #.#, expect_helpers, timeout_but_pass, failed, Test case was expected to hang but it continued execution 137tp, #.#, expect_helpers, failed 138EOF 139# NO_CHECK_STYLE_END 140 atf_check -s eq:0 -o file:expout -e empty -x \ 141 "atf-report -o csv:- <tps.out | " \ 142 "sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'" 143 144# NO_CHECK_STYLE_BEGIN 145 cat >expout <<EOF 146expect_helpers (1/1): 17 test cases 147 death_and_exit: [#.#s] Expected failure: Exit case 148 death_and_signal: [#.#s] Expected failure: Signal case 149 death_but_pass: [#.#s] Failed: Test case was expected to terminate abruptly but it continued execution 150 exit_any_and_exit: [#.#s] Expected failure: Call will exit 151 exit_but_pass: [#.#s] Failed: Test case was expected to exit cleanly but it continued execution 152 exit_code_and_exit: [#.#s] Expected failure: Call will exit 153 fail_and_fail_check: [#.#s] Expected failure: And fail again: 2 checks failed as expected; see output for more details 154 fail_and_fail_requirement: [#.#s] Expected failure: Fail reason: The failure 155 fail_but_pass: [#.#s] Failed: Test case was expecting a failure but none were raised 156 pass_and_pass: [#.#s] Passed. 157 pass_but_fail_check: [#.#s] Failed: 1 checks failed; see output for more details 158 pass_but_fail_requirement: [#.#s] Failed: Some reason 159 signal_any_and_signal: [#.#s] Expected failure: Call will signal 160 signal_but_pass: [#.#s] Failed: Test case was expected to receive a termination signal but it continued execution 161 signal_no_and_signal: [#.#s] Expected failure: Call will signal 162 timeout_and_hang: [#.#s] Expected failure: Will overrun 163 timeout_but_pass: [#.#s] Failed: Test case was expected to hang but it continued execution 164[#.#s] 165 166Test cases for known bugs: 167 expect_helpers:death_and_exit: Exit case 168 expect_helpers:death_and_signal: Signal case 169 expect_helpers:exit_any_and_exit: Call will exit 170 expect_helpers:exit_code_and_exit: Call will exit 171 expect_helpers:fail_and_fail_check: And fail again: 2 checks failed as expected; see output for more details 172 expect_helpers:fail_and_fail_requirement: Fail reason: The failure 173 expect_helpers:signal_any_and_signal: Call will signal 174 expect_helpers:signal_no_and_signal: Call will signal 175 expect_helpers:timeout_and_hang: Will overrun 176 177Failed test cases: 178 expect_helpers:death_but_pass, expect_helpers:exit_but_pass, expect_helpers:fail_but_pass, expect_helpers:pass_but_fail_check, expect_helpers:pass_but_fail_requirement, expect_helpers:signal_but_pass, expect_helpers:timeout_but_pass 179 180Summary for 1 test programs: 181 1 passed test cases. 182 7 failed test cases. 183 9 expected failed test cases. 184 0 skipped test cases. 185EOF 186# NO_CHECK_STYLE_END 187 atf_check -s eq:0 -o file:expout -e empty -x \ 188 "atf-report -o ticker:- <tps.out | " \ 189 "sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'" 190 191 # Just ensure that this does not crash for now... 192 atf_check -s eq:0 -o ignore -e empty -x "atf-report -o xml:- <tps.out" 193} 194 195atf_test_case oflag 196oflag_head() 197{ 198 atf_set "descr" "Checks that the -o flag works" 199} 200oflag_body() 201{ 202 create_helpers 203 run_helpers 204 205 # Get the default output. 206 atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report <tps.out' 207 mv stdout defout 208 209 # Check that changing the stdout output works. 210 atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report -o csv:- <tps.out' 211 atf_check -s eq:1 -o empty -e empty cmp -s defout stdout 212 cp stdout expcsv 213 214 # Check that sending the output to a file does not write to stdout. 215 atf_check -s eq:0 -o empty -e empty -x 'atf-report -o csv:fmt.out <tps.out' 216 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out 217 rm -f fmt.out 218 219 # Check that defining two outputs using the same format works. 220 atf_check -s eq:0 -o empty -e empty -x \ 221 'atf-report -o csv:fmt.out -o csv:fmt2.out <tps.out' 222 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out 223 atf_check -s eq:0 -o empty -e empty cmp -s fmt.out fmt2.out 224 rm -f fmt.out fmt2.out 225 226 # Check that defining two outputs using different formats works. 227 atf_check -s eq:0 -o empty -e empty -x \ 228 'atf-report -o csv:fmt.out -o ticker:fmt2.out <tps.out' 229 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out 230 atf_check -s eq:1 -o empty -e empty cmp -s fmt.out fmt2.out 231 atf_check -s eq:0 -o ignore -e empty grep "test cases" fmt2.out 232 atf_check -s eq:0 -o ignore -e empty grep "Failed test cases" fmt2.out 233 atf_check -s eq:0 -o ignore -e empty grep "Summary for" fmt2.out 234 rm -f fmt.out fmt2.out 235 236 # Check that defining two outputs over the same file does not work. 237 atf_check -s eq:1 -o empty -e match:'more than once' -x \ 238 'atf-report -o csv:fmt.out -o ticker:fmt.out <tps.out' 239 rm -f fmt.out 240 241 # Check that defining two outputs over stdout (but using different 242 # paths) does not work. 243 atf_check -s eq:1 -o empty -e match:'more than once' -x \ 244 'atf-report -o csv:- -o ticker:/dev/stdout <tps.out' 245 rm -f fmt.out 246} 247 248atf_test_case output_csv 249output_csv_head() 250{ 251 atf_set "descr" "Checks the CSV output format" 252} 253output_csv_body() 254{ 255 create_helpers 256 run_helpers 257 258# NO_CHECK_STYLE_BEGIN 259 cat >expout <<EOF 260tc, #.#, dir1/tp1, main, passed 261tp, #.#, dir1/tp1, passed 262tc, #.#, dir1/tp2, main, failed, This always fails 263tp, #.#, dir1/tp2, failed 264tc, #.#, tp3, main, passed 265tp, #.#, tp3, passed 266tc, #.#, tp4, main, failed, This always fails 267tp, #.#, tp4, failed 268tp, #.#, tp5, bogus, Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:' 269EOF 270# NO_CHECK_STYLE_END 271 272 atf_check -s eq:0 -o file:expout -e empty -x \ 273 "atf-report -o csv:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'" 274} 275 276atf_test_case output_ticker 277output_ticker_head() 278{ 279 atf_set "descr" "Checks the ticker output format" 280} 281output_ticker_body() 282{ 283 create_helpers 284 run_helpers 285 286# NO_CHECK_STYLE_BEGIN 287 cat >expout <<EOF 288dir1/tp1 (1/5): 1 test cases 289 main: [#.#s] Passed. 290[#.#s] 291 292dir1/tp2 (2/5): 1 test cases 293 main: [#.#s] Failed: This always fails 294[#.#s] 295 296tp3 (3/5): 1 test cases 297 main: [#.#s] Passed. 298[#.#s] 299 300tp4 (4/5): 1 test cases 301 main: [#.#s] Failed: This always fails 302[#.#s] 303 304tp5 (5/5): 0 test cases 305tp5: BOGUS TEST PROGRAM: Cannot trust its results because of \`Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'' 306[#.#s] 307 308Failed (bogus) test programs: 309 tp5 310 311Failed test cases: 312 dir1/tp2:main, tp4:main 313 314Summary for 5 test programs: 315 2 passed test cases. 316 2 failed test cases. 317 0 expected failed test cases. 318 0 skipped test cases. 319EOF 320 321 atf_check -s eq:0 -o file:expout -e empty -x \ 322 "atf-report -o ticker:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'" 323} 324# NO_CHECK_STYLE_END 325 326atf_test_case output_xml 327output_xml_head() 328{ 329 atf_set "descr" "Checks the XML output format" 330} 331output_xml_body() 332{ 333 create_helpers 334 run_helpers 335 336# NO_CHECK_STYLE_BEGIN 337 cat >expout <<EOF 338<?xml version="1.0" encoding="ISO-8859-1"?> 339<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd"> 340 341<tests-results> 342<info class="startinfo">A value</info> 343<tp id="dir1/tp1"> 344<tc id="main"> 345<passed /> 346<tc-time>#.#</tc-time></tc> 347<tp-time>#.#</tp-time></tp> 348<tp id="dir1/tp2"> 349<tc id="main"> 350<failed>This always fails</failed> 351<tc-time>#.#</tc-time></tc> 352<tp-time>#.#</tp-time></tp> 353<tp id="tp3"> 354<tc id="main"> 355<passed /> 356<tc-time>#.#</tc-time></tc> 357<tp-time>#.#</tp-time></tp> 358<tp id="tp4"> 359<tc id="main"> 360<failed>This always fails</failed> 361<tc-time>#.#</tc-time></tc> 362<tp-time>#.#</tp-time></tp> 363<tp id="tp5"> 364<failed>Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'</failed> 365<tp-time>#.#</tp-time></tp> 366<info class="endinfo">Another value</info> 367</tests-results> 368EOF 369# NO_CHECK_STYLE_END 370 371 atf_check -s eq:0 -o file:expout -e empty -x \ 372 "atf-report -o xml:- < tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'" 373} 374 375atf_test_case output_xml_space 376output_xml_space_head() 377{ 378 atf_set "descr" "Checks that the XML output format properly preserves" \ 379 "leading and trailing whitespace in stdout and stderr" \ 380 "lines" 381} 382output_xml_space_body() 383{ 384 cp $(atf_get_srcdir)/misc_helpers . 385 cat >Atffile <<EOF 386Content-Type: application/X-atf-atffile; version="1" 387 388prop: test-suite = atf 389 390tp: misc_helpers 391EOF 392 393# NO_CHECK_STYLE_BEGIN 394 cat >expout <<EOF 395<?xml version="1.0" encoding="ISO-8859-1"?> 396<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd"> 397 398<tests-results> 399<info class="startinfo">A value</info> 400<tp id="misc_helpers"> 401<tc id="diff"> 402<so>--- a 2007-11-04 14:00:41.000000000 +0100</so> 403<so>+++ b 2007-11-04 14:00:48.000000000 +0100</so> 404<so>@@ -1,7 +1,7 @@</so> 405<so> This test is meant to simulate a diff.</so> 406<so> Blank space at beginning of context lines must be preserved.</so> 407<so> </so> 408<so>-First original line.</so> 409<so>-Second original line.</so> 410<so>+First modified line.</so> 411<so>+Second modified line.</so> 412<so> </so> 413<so> EOF</so> 414<passed /> 415<tc-time>#.#</tc-time></tc> 416<tp-time>#.#</tp-time></tp> 417<info class="endinfo">Another value</info> 418</tests-results> 419EOF 420# NO_CHECK_STYLE_END 421 422 run_helpers 423 atf_check -s eq:0 -o file:expout -e empty -x \ 424 "atf-report -o xml:- <tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'" 425} 426 427atf_test_case too_many_args 428too_many_args_body() 429{ 430 cat >experr <<EOF 431atf-report: ERROR: No arguments allowed 432EOF 433 atf_check -s eq:1 -o empty -e file:experr atf-report foo 434} 435 436atf_init_test_cases() 437{ 438 atf_add_test_case default 439 atf_add_test_case expect 440 atf_add_test_case oflag 441 atf_add_test_case output_csv 442 atf_add_test_case output_ticker 443 atf_add_test_case output_xml 444 atf_add_test_case output_xml_space 445 atf_add_test_case too_many_args 446} 447 448# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4 449