1#! __SHTK_SHELL__ 2# Copyright 2014 Google Inc. 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# * Neither the name of Google Inc. nor the names of its contributors 15# may be used to endorse or promote products derived from this software 16# without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30# Bootstrap and integration tests for the "unittest" module. 31# 32# This file implements a bunch of rudimentary tests for the "unittest" 33# module and serve two purposes: first, that test programs using "unittest" 34# and built via shtk are runnable and execute all defined test cases; and, 35# second, that the pass/fail code of the test program matches the results 36# of the individual test cases. 37# 38# This test program is purposely not written using shtk's modules at all 39# precisely to ensure that the shtk+unittest combination works. Only once 40# we have certain confidence in the behavior of shtk-based test programs, 41# we can trust the results yielded by the more complex and detailed tests 42# in unittest_test.sh. 43 44 45# The program name for error-reporting purposes. 46_ProgName="${0##*/}" 47 48 49# Logs an error and fails the test program. 50# 51# \param ... Error message. Multiple arguments are concatenated using a 52# single whitespace character. 53fail() { 54 echo "${_ProgName}: ${*}" 1>&2 55 exit 1 56} 57 58 59# Executes a command and validates its exit code and its output. 60# 61# \param expected_exit_code Expected code that the command should return. 62# \param expected_stdout_file Path to a file containing the expected stdout. 63# \param expected_stderr_file Path to a file containing the expected stderr. 64# \param ... The command to execute. 65# 66# \return True if the command's execution matches the expected conditions; 67# false otherwise. 68check() { 69 local expected_exit_code="${1}"; shift 70 local expected_stdout_file="${1}"; shift 71 local expected_stderr_file="${1}"; shift 72 73 local exit_code=0 74 "${@}" >out 2>err || exit_code="${?}" 75 76 local failed=no 77 [ "${exit_code}" -eq "${expected_exit_code}" ] || failed=yes 78 cmp -s "${expected_stdout_file}" out || failed=yes 79 cmp -s "${expected_stderr_file}" err || failed=yes 80 81 if [ "${failed}" = yes ]; then 82 echo "${@} failed" 83 echo "Expected exit code ${expected_exit_code}, got ${exit_code}" 84 diff -u "${expected_stdout_file}" out 85 diff -u "${expected_stderr_file}" err 86 return 1 87 fi 88} 89 90 91one_test__always_passes() { 92 shtk build -m shtk_unittest_main -o program - <<EOF 93shtk_import unittest 94 95shtk_unittest_add_test always_passes 96always_passes_test() { 97 echo "Hello" 98} 99EOF 100 101 cat >expout <<EOF 102Hello 103EOF 104 105 cat >experr <<EOF 106program: I: Testing always_passes... 107program: I: Testing always_passes... PASSED 108program: I: Ran 1 tests; ALL PASSED 109EOF 110 111 check 0 expout experr ./program || return 1 112} 113 114 115one_test__always_fails() { 116 shtk build -m shtk_unittest_main -o program - <<EOF 117shtk_import unittest 118 119shtk_unittest_add_test always_fails 120always_fails_test() { 121 echo "Hello" 122 fail "Oops! Explicitly failing" 123 echo "Bye" 124} 125EOF 126 127 cat >expout <<EOF 128Hello 129EOF 130 131 cat >experr <<EOF 132program: I: Testing always_fails... 133program: E: Oops! Explicitly failing 134program: W: Testing always_fails... FAILED 135program: W: Ran 1 tests; 1 FAILED 136EOF 137 138 check 1 expout experr ./program || return 1 139} 140 141 142some_tests__all_pass() { 143 shtk build -m shtk_unittest_main -o program - <<EOF 144shtk_import unittest 145 146shtk_unittest_add_test first 147first_test() { echo "First"; } 148 149shtk_unittest_add_test second 150second_test() { set_expect_failure; fail "Second"; echo "not reached"; } 151 152shtk_unittest_add_test third 153third_test() { echo "Third"; } 154EOF 155 156 cat >expout <<EOF 157First 158Third 159EOF 160 161 cat >experr <<EOF 162program: I: Testing first... 163program: I: Testing first... PASSED 164program: I: Testing second... 165program: E: Expected failure: Second 166program: I: Testing second... EXPECTED FAILURE 167program: I: Testing third... 168program: I: Testing third... PASSED 169program: I: Ran 3 tests; ALL PASSED 170EOF 171 172 check 0 expout experr ./program || return 1 173} 174 175 176some_tests__some_fail() { 177 shtk build -m shtk_unittest_main -o program - <<EOF 178shtk_import unittest 179 180shtk_unittest_add_test first 181first_test() { echo "First"; } 182 183shtk_unittest_add_test second 184second_test() { echo "Second"; fail "Bailing out"; echo "Second bis"; } 185 186shtk_unittest_add_test third 187third_test() { echo "Third"; } 188EOF 189 190 cat >expout <<EOF 191First 192Second 193Third 194EOF 195 196 cat >experr <<EOF 197program: I: Testing first... 198program: I: Testing first... PASSED 199program: I: Testing second... 200program: E: Bailing out 201program: W: Testing second... FAILED 202program: I: Testing third... 203program: I: Testing third... PASSED 204program: W: Ran 3 tests; 1 FAILED 205EOF 206 207 check 1 expout experr ./program || return 1 208} 209 210 211fixtures() { 212 shtk build -m shtk_unittest_main -o program - <<EOF 213shtk_import unittest 214 215shtk_unittest_add_fixture first 216first_fixture() { 217 setup() { echo "Shared setup"; } 218 teardown() { echo "Shared teardown"; } 219 shtk_unittest_add_test first 220 first_test() { echo "First in first fixture"; } 221 shtk_unittest_add_test second 222 second_test() { echo "Second in first fixture"; fail "Leave 1"; } 223} 224 225shtk_unittest_add_fixture second 226second_fixture() { 227 shtk_unittest_add_test first 228 first_test() { echo "First in second fixture"; fail "Leave 2"; } 229 shtk_unittest_add_test second 230 second_test() { echo "Second in second fixture"; } 231} 232 233shtk_unittest_add_test standalone 234standalone_test() { 235 echo "Runs outside of the fixtures" 236} 237EOF 238 239 cat >expout <<EOF 240Runs outside of the fixtures 241Shared setup 242First in first fixture 243Shared teardown 244Shared setup 245Second in first fixture 246Shared teardown 247First in second fixture 248Second in second fixture 249EOF 250 251 cat >experr <<EOF 252program: I: Testing standalone... 253program: I: Testing standalone... PASSED 254program: I: Testing first__first... 255program: I: Testing first__first... PASSED 256program: I: Testing first__second... 257program: E: Leave 1 258program: W: Testing first__second... FAILED 259program: I: Testing second__first... 260program: E: Leave 2 261program: W: Testing second__first... FAILED 262program: I: Testing second__second... 263program: I: Testing second__second... PASSED 264program: W: Ran 5 tests; 2 FAILED 265EOF 266 267 check 1 expout experr ./program || return 1 268} 269 270 271assert_command__ok() { 272 cat >command.sh <<EOF 273echo "some contents to stdout" 274echo "some contents to stderr" 1>&2 275exit 42 276EOF 277 278 shtk build -m shtk_unittest_main -o program - <<EOF 279shtk_import unittest 280 281shtk_unittest_add_test first 282first_test() { 283 echo "some contents to stdout" >expout 284 echo "some contents to stderr" >experr 285 assert_command -s exit:42 -o file:expout -e file:experr sh $(pwd)/command.sh 286 echo "reached" 287} 288EOF 289 290 cat >expout <<EOF 291Running checked command: sh $(pwd)/command.sh 292reached 293EOF 294 295 cat >experr <<EOF 296program: I: Testing first... 297program: I: Testing first... PASSED 298program: I: Ran 1 tests; ALL PASSED 299EOF 300 301 check 0 expout experr ./program || return 1 302} 303 304 305assert_command__fail() { 306 cat >command.sh <<EOF 307echo "some contents to stdout" 308echo "some contents to stderr" 1>&2 309exit 42 310EOF 311 312 shtk build -m shtk_unittest_main -o program - <<EOF 313shtk_import unittest 314 315shtk_unittest_add_test first 316first_test() { 317 assert_command -s exit:42 sh $(pwd)/command.sh 318 echo "not reached" 319} 320EOF 321 322 cat >expout <<EOF 323Running checked command: sh $(pwd)/command.sh 324Expected standard output to be empty; found: 325some contents to stdout 326Expected standard error to be empty; found: 327some contents to stderr 328EOF 329 330 cat >experr <<EOF 331program: I: Testing first... 332program: E: Check of 'sh $(pwd)/command.sh' failed; see stdout for details 333program: W: Testing first... FAILED 334program: W: Ran 1 tests; 1 FAILED 335EOF 336 337 check 1 expout experr ./program || return 1 338} 339 340 341expect_command__ok() { 342 cat >command.sh <<EOF 343echo "some contents to stdout" 344echo "some contents to stderr" 1>&2 345exit 42 346EOF 347 348 shtk build -m shtk_unittest_main -o program - <<EOF 349shtk_import unittest 350 351shtk_unittest_add_test first 352first_test() { 353 echo "some contents to stdout" >expout 354 echo "some contents to stderr" >experr 355 expect_command -s exit:42 -o file:expout -e file:experr sh $(pwd)/command.sh 356 echo "reached" 357} 358EOF 359 360 cat >expout <<EOF 361Running checked command: sh $(pwd)/command.sh 362reached 363EOF 364 365 cat >experr <<EOF 366program: I: Testing first... 367program: I: Testing first... PASSED 368program: I: Ran 1 tests; ALL PASSED 369EOF 370 371 check 0 expout experr ./program || return 1 372} 373 374 375expect_command__fail() { 376 cat >command.sh <<EOF 377echo "some contents to stdout" 378echo "some contents to stderr" 1>&2 379exit 42 380EOF 381 382 shtk build -m shtk_unittest_main -o program - <<EOF 383shtk_import unittest 384 385shtk_unittest_add_test first 386first_test() { 387 expect_command -s exit:42 sh $(pwd)/command.sh 388 echo "reached" 389 expect_command sh $(pwd)/command.sh 390 echo "also reached" 391} 392EOF 393 394 cat >expout <<EOF 395Running checked command: sh $(pwd)/command.sh 396Expected standard output to be empty; found: 397some contents to stdout 398Expected standard error to be empty; found: 399some contents to stderr 400reached 401Running checked command: sh $(pwd)/command.sh 402Expected exit code 0 != actual exit code 42 403stdout: some contents to stdout 404stderr: some contents to stderr 405also reached 406EOF 407 408 cat >experr <<EOF 409program: I: Testing first... 410program: W: Delayed failure: Check of 'sh $(pwd)/command.sh' failed; see stdout for details 411program: W: Delayed failure: Check of 'sh $(pwd)/command.sh' failed; see stdout for details 412program: W: Testing first... FAILED (2 delayed failures) 413program: W: Ran 1 tests; 1 FAILED 414EOF 415 416 check 1 expout experr ./program || return 1 417} 418 419 420assert_file__ok() { 421 shtk build -m shtk_unittest_main -o program - <<EOF 422shtk_import unittest 423 424shtk_unittest_add_test first 425first_test() { 426 touch actual 427 assert_file empty actual 428 429 echo "foo" >>actual 430 echo "bar" >>actual 431 assert_file not-empty actual 432 433 assert_file inline:"foo\nbar\n" actual 434 435 assert_file file:actual actual 436 assert_file stdin actual <actual 437 438 echo "reached" 439} 440EOF 441 442 cat >expout <<EOF 443reached 444EOF 445 446 cat >experr <<EOF 447program: I: Testing first... 448program: I: Testing first... PASSED 449program: I: Ran 1 tests; ALL PASSED 450EOF 451 452 check 0 expout experr ./program || return 1 453} 454 455 456assert_file__fail() { 457 shtk build -m shtk_unittest_main -o program - <<EOF 458shtk_import unittest 459 460shtk_unittest_add_test first 461first_test() { 462 touch actual 463 assert_file not-empty actual 464 echo reached 465} 466 467shtk_unittest_add_test second 468second_test() { 469 echo "foo" >actual 470 assert_file empty actual 471 echo reached 472} 473 474shtk_unittest_add_test third 475third_test() { 476 echo "foo" >actual 477 assert_file match:"foobar" actual 478 echo reached 479} 480EOF 481 482 cat >expout <<EOF 483Expected actual to not be empty 484Expected actual to be empty; found: 485foo 486Expected regexp 'foobar' not found in actual: 487foo 488EOF 489 490 cat >experr <<EOF 491program: I: Testing first... 492program: E: Failed to validate contents of file actual 493program: W: Testing first... FAILED 494program: I: Testing second... 495program: E: Failed to validate contents of file actual 496program: W: Testing second... FAILED 497program: I: Testing third... 498program: E: Failed to validate contents of file actual 499program: W: Testing third... FAILED 500program: W: Ran 3 tests; 3 FAILED 501EOF 502 503 check 1 expout experr ./program || return 1 504} 505 506 507expect_file__ok() { 508 shtk build -m shtk_unittest_main -o program - <<EOF 509shtk_import unittest 510 511shtk_unittest_add_test first 512first_test() { 513 touch actual 514 expect_file empty actual 515 516 echo "foo" >>actual 517 echo "bar" >>actual 518 expect_file not-empty actual 519 520 expect_file inline:"foo\nbar\n" actual 521 522 expect_file file:actual actual 523 expect_file stdin actual <actual 524 525 echo "reached" 526} 527EOF 528 529 cat >expout <<EOF 530reached 531EOF 532 533 cat >experr <<EOF 534program: I: Testing first... 535program: I: Testing first... PASSED 536program: I: Ran 1 tests; ALL PASSED 537EOF 538 539 check 0 expout experr ./program || return 1 540} 541 542 543expect_file__fail() { 544 shtk build -m shtk_unittest_main -o program - <<EOF 545shtk_import unittest 546 547shtk_unittest_add_test first 548first_test() { 549 touch actual 550 expect_file not-empty actual 551 552 echo "foo" >>actual 553 echo "bar" >>actual 554 expect_file empty actual 555 556 expect_file match:foobar actual 557 558 echo "reached" 559} 560EOF 561 562 cat >expout <<EOF 563Expected actual to not be empty 564Expected actual to be empty; found: 565foo 566bar 567Expected regexp 'foobar' not found in actual: 568foo 569bar 570reached 571EOF 572 573 cat >experr <<EOF 574program: I: Testing first... 575program: W: Delayed failure: Failed to validate contents of file actual 576program: W: Delayed failure: Failed to validate contents of file actual 577program: W: Delayed failure: Failed to validate contents of file actual 578program: W: Testing first... FAILED (3 delayed failures) 579program: W: Ran 1 tests; 1 FAILED 580EOF 581 582 check 1 expout experr ./program || return 1 583} 584 585 586main() { 587 for name in \ 588 one_test__always_passes \ 589 one_test__always_fails \ 590 some_tests__all_pass \ 591 some_tests__some_fail \ 592 fixtures \ 593 assert_command__ok \ 594 assert_command__fail \ 595 expect_command__ok \ 596 expect_command__fail \ 597 assert_file__ok \ 598 assert_file__fail \ 599 expect_file__ok \ 600 expect_file__fail 601 do 602 local failed=no 603 echo "Running test ${name}" 604 ( 605 mkdir work 606 cd work 607 "${name}" 608 ) || failed=yes 609 rm -rf work 610 [ "${failed}" = no ] || fail "unittest is severely broken; aborting!" 611 done 612 echo "OK!" 613} 614 615 616main "${@}" 617