1# 2# subunit: extensions to python unittest to get test results from subprocesses. 3# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net> 4# 5# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause 6# license at the users choice. A copy of both licenses are available in the 7# project source as Apache-2.0 and BSD. You may not use this file except in 8# compliance with one of these two licences. 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# license you chose for the specific language governing permissions and 14# limitations under that license. 15# 16 17"""Tests for TAP2SubUnit.""" 18 19import unittest 20 21from testtools.compat import StringIO 22 23import subunit 24 25 26class TestTAP2SubUnit(unittest.TestCase): 27 """Tests for TAP2SubUnit. 28 29 These tests test TAP string data in, and subunit string data out. 30 This is ok because the subunit protocol is intended to be stable, 31 but it might be easier/pithier to write tests against TAP string in, 32 parsed subunit objects out (by hooking the subunit stream to a subunit 33 protocol server. 34 """ 35 36 def setUp(self): 37 self.tap = StringIO() 38 self.subunit = StringIO() 39 40 def test_skip_entire_file(self): 41 # A file 42 # 1..- # Skipped: comment 43 # results in a single skipped test. 44 self.tap.write("1..0 # Skipped: entire file skipped\n") 45 self.tap.seek(0) 46 result = subunit.TAP2SubUnit(self.tap, self.subunit) 47 self.assertEqual(0, result) 48 self.assertEqual([ 49 "test file skip", 50 "skip file skip [", 51 "Skipped: entire file skipped", 52 "]", 53 ], 54 self.subunit.getvalue().splitlines()) 55 56 def test_ok_test_pass(self): 57 # A file 58 # ok 59 # results in a passed test with name 'test 1' (a synthetic name as tap 60 # does not require named fixtures - it is the first test in the tap 61 # stream). 62 self.tap.write("ok\n") 63 self.tap.seek(0) 64 result = subunit.TAP2SubUnit(self.tap, self.subunit) 65 self.assertEqual(0, result) 66 self.assertEqual([ 67 "test test 1", 68 "success test 1", 69 ], 70 self.subunit.getvalue().splitlines()) 71 72 def test_ok_test_number_pass(self): 73 # A file 74 # ok 1 75 # results in a passed test with name 'test 1' 76 self.tap.write("ok 1\n") 77 self.tap.seek(0) 78 result = subunit.TAP2SubUnit(self.tap, self.subunit) 79 self.assertEqual(0, result) 80 self.assertEqual([ 81 "test test 1", 82 "success test 1", 83 ], 84 self.subunit.getvalue().splitlines()) 85 86 def test_ok_test_number_description_pass(self): 87 # A file 88 # ok 1 - There is a description 89 # results in a passed test with name 'test 1 - There is a description' 90 self.tap.write("ok 1 - There is a description\n") 91 self.tap.seek(0) 92 result = subunit.TAP2SubUnit(self.tap, self.subunit) 93 self.assertEqual(0, result) 94 self.assertEqual([ 95 "test test 1 - There is a description", 96 "success test 1 - There is a description", 97 ], 98 self.subunit.getvalue().splitlines()) 99 100 def test_ok_test_description_pass(self): 101 # A file 102 # ok There is a description 103 # results in a passed test with name 'test 1 There is a description' 104 self.tap.write("ok There is a description\n") 105 self.tap.seek(0) 106 result = subunit.TAP2SubUnit(self.tap, self.subunit) 107 self.assertEqual(0, result) 108 self.assertEqual([ 109 "test test 1 There is a description", 110 "success test 1 There is a description", 111 ], 112 self.subunit.getvalue().splitlines()) 113 114 def test_ok_SKIP_skip(self): 115 # A file 116 # ok # SKIP 117 # results in a skkip test with name 'test 1' 118 self.tap.write("ok # SKIP\n") 119 self.tap.seek(0) 120 result = subunit.TAP2SubUnit(self.tap, self.subunit) 121 self.assertEqual(0, result) 122 self.assertEqual([ 123 "test test 1", 124 "skip test 1", 125 ], 126 self.subunit.getvalue().splitlines()) 127 128 def test_ok_skip_number_comment_lowercase(self): 129 self.tap.write("ok 1 # skip no samba environment available, skipping compilation\n") 130 self.tap.seek(0) 131 result = subunit.TAP2SubUnit(self.tap, self.subunit) 132 self.assertEqual(0, result) 133 self.assertEqual([ 134 "test test 1", 135 "skip test 1 [", 136 "no samba environment available, skipping compilation", 137 "]" 138 ], 139 self.subunit.getvalue().splitlines()) 140 141 def test_ok_number_description_SKIP_skip_comment(self): 142 # A file 143 # ok 1 foo # SKIP Not done yet 144 # results in a skip test with name 'test 1 foo' and a log of 145 # Not done yet 146 self.tap.write("ok 1 foo # SKIP Not done yet\n") 147 self.tap.seek(0) 148 result = subunit.TAP2SubUnit(self.tap, self.subunit) 149 self.assertEqual(0, result) 150 self.assertEqual([ 151 "test test 1 foo", 152 "skip test 1 foo [", 153 "Not done yet", 154 "]", 155 ], 156 self.subunit.getvalue().splitlines()) 157 158 def test_ok_SKIP_skip_comment(self): 159 # A file 160 # ok # SKIP Not done yet 161 # results in a skip test with name 'test 1' and a log of Not done yet 162 self.tap.write("ok # SKIP Not done yet\n") 163 self.tap.seek(0) 164 result = subunit.TAP2SubUnit(self.tap, self.subunit) 165 self.assertEqual(0, result) 166 self.assertEqual([ 167 "test test 1", 168 "skip test 1 [", 169 "Not done yet", 170 "]", 171 ], 172 self.subunit.getvalue().splitlines()) 173 174 def test_ok_TODO_xfail(self): 175 # A file 176 # ok # TODO 177 # results in a xfail test with name 'test 1' 178 self.tap.write("ok # TODO\n") 179 self.tap.seek(0) 180 result = subunit.TAP2SubUnit(self.tap, self.subunit) 181 self.assertEqual(0, result) 182 self.assertEqual([ 183 "test test 1", 184 "xfail test 1", 185 ], 186 self.subunit.getvalue().splitlines()) 187 188 def test_ok_TODO_xfail_comment(self): 189 # A file 190 # ok # TODO Not done yet 191 # results in a xfail test with name 'test 1' and a log of Not done yet 192 self.tap.write("ok # TODO Not done yet\n") 193 self.tap.seek(0) 194 result = subunit.TAP2SubUnit(self.tap, self.subunit) 195 self.assertEqual(0, result) 196 self.assertEqual([ 197 "test test 1", 198 "xfail test 1 [", 199 "Not done yet", 200 "]", 201 ], 202 self.subunit.getvalue().splitlines()) 203 204 def test_bail_out_errors(self): 205 # A file with line in it 206 # Bail out! COMMENT 207 # is treated as an error 208 self.tap.write("ok 1 foo\n") 209 self.tap.write("Bail out! Lifejacket engaged\n") 210 self.tap.seek(0) 211 result = subunit.TAP2SubUnit(self.tap, self.subunit) 212 self.assertEqual(0, result) 213 self.assertEqual([ 214 "test test 1 foo", 215 "success test 1 foo", 216 "test Bail out! Lifejacket engaged", 217 "error Bail out! Lifejacket engaged", 218 ], 219 self.subunit.getvalue().splitlines()) 220 221 def test_missing_test_at_end_with_plan_adds_error(self): 222 # A file 223 # 1..3 224 # ok first test 225 # not ok third test 226 # results in three tests, with the third being created 227 self.tap.write('1..3\n') 228 self.tap.write('ok first test\n') 229 self.tap.write('not ok second test\n') 230 self.tap.seek(0) 231 result = subunit.TAP2SubUnit(self.tap, self.subunit) 232 self.assertEqual(0, result) 233 self.assertEqual([ 234 'test test 1 first test', 235 'success test 1 first test', 236 'test test 2 second test', 237 'failure test 2 second test', 238 'test test 3', 239 'error test 3 [', 240 'test missing from TAP output', 241 ']', 242 ], 243 self.subunit.getvalue().splitlines()) 244 245 def test_missing_test_with_plan_adds_error(self): 246 # A file 247 # 1..3 248 # ok first test 249 # not ok 3 third test 250 # results in three tests, with the second being created 251 self.tap.write('1..3\n') 252 self.tap.write('ok first test\n') 253 self.tap.write('not ok 3 third test\n') 254 self.tap.seek(0) 255 result = subunit.TAP2SubUnit(self.tap, self.subunit) 256 self.assertEqual(0, result) 257 self.assertEqual([ 258 'test test 1 first test', 259 'success test 1 first test', 260 'test test 2', 261 'error test 2 [', 262 'test missing from TAP output', 263 ']', 264 'test test 3 third test', 265 'failure test 3 third test', 266 ], 267 self.subunit.getvalue().splitlines()) 268 269 def test_missing_test_no_plan_adds_error(self): 270 # A file 271 # ok first test 272 # not ok 3 third test 273 # results in three tests, with the second being created 274 self.tap.write('ok first test\n') 275 self.tap.write('not ok 3 third test\n') 276 self.tap.seek(0) 277 result = subunit.TAP2SubUnit(self.tap, self.subunit) 278 self.assertEqual(0, result) 279 self.assertEqual([ 280 'test test 1 first test', 281 'success test 1 first test', 282 'test test 2', 283 'error test 2 [', 284 'test missing from TAP output', 285 ']', 286 'test test 3 third test', 287 'failure test 3 third test', 288 ], 289 self.subunit.getvalue().splitlines()) 290 291 def test_four_tests_in_a_row_trailing_plan(self): 292 # A file 293 # ok 1 - first test in a script with no plan at all 294 # not ok 2 - second 295 # ok 3 - third 296 # not ok 4 - fourth 297 # 1..4 298 # results in four tests numbered and named 299 self.tap.write('ok 1 - first test in a script with trailing plan\n') 300 self.tap.write('not ok 2 - second\n') 301 self.tap.write('ok 3 - third\n') 302 self.tap.write('not ok 4 - fourth\n') 303 self.tap.write('1..4\n') 304 self.tap.seek(0) 305 result = subunit.TAP2SubUnit(self.tap, self.subunit) 306 self.assertEqual(0, result) 307 self.assertEqual([ 308 'test test 1 - first test in a script with trailing plan', 309 'success test 1 - first test in a script with trailing plan', 310 'test test 2 - second', 311 'failure test 2 - second', 312 'test test 3 - third', 313 'success test 3 - third', 314 'test test 4 - fourth', 315 'failure test 4 - fourth' 316 ], 317 self.subunit.getvalue().splitlines()) 318 319 def test_four_tests_in_a_row_with_plan(self): 320 # A file 321 # 1..4 322 # ok 1 - first test in a script with no plan at all 323 # not ok 2 - second 324 # ok 3 - third 325 # not ok 4 - fourth 326 # results in four tests numbered and named 327 self.tap.write('1..4\n') 328 self.tap.write('ok 1 - first test in a script with a plan\n') 329 self.tap.write('not ok 2 - second\n') 330 self.tap.write('ok 3 - third\n') 331 self.tap.write('not ok 4 - fourth\n') 332 self.tap.seek(0) 333 result = subunit.TAP2SubUnit(self.tap, self.subunit) 334 self.assertEqual(0, result) 335 self.assertEqual([ 336 'test test 1 - first test in a script with a plan', 337 'success test 1 - first test in a script with a plan', 338 'test test 2 - second', 339 'failure test 2 - second', 340 'test test 3 - third', 341 'success test 3 - third', 342 'test test 4 - fourth', 343 'failure test 4 - fourth' 344 ], 345 self.subunit.getvalue().splitlines()) 346 347 def test_four_tests_in_a_row_no_plan(self): 348 # A file 349 # ok 1 - first test in a script with no plan at all 350 # not ok 2 - second 351 # ok 3 - third 352 # not ok 4 - fourth 353 # results in four tests numbered and named 354 self.tap.write('ok 1 - first test in a script with no plan at all\n') 355 self.tap.write('not ok 2 - second\n') 356 self.tap.write('ok 3 - third\n') 357 self.tap.write('not ok 4 - fourth\n') 358 self.tap.seek(0) 359 result = subunit.TAP2SubUnit(self.tap, self.subunit) 360 self.assertEqual(0, result) 361 self.assertEqual([ 362 'test test 1 - first test in a script with no plan at all', 363 'success test 1 - first test in a script with no plan at all', 364 'test test 2 - second', 365 'failure test 2 - second', 366 'test test 3 - third', 367 'success test 3 - third', 368 'test test 4 - fourth', 369 'failure test 4 - fourth' 370 ], 371 self.subunit.getvalue().splitlines()) 372 373 def test_todo_and_skip(self): 374 # A file 375 # not ok 1 - a fail but # TODO but is TODO 376 # not ok 2 - another fail # SKIP instead 377 # results in two tests, numbered and commented. 378 self.tap.write("not ok 1 - a fail but # TODO but is TODO\n") 379 self.tap.write("not ok 2 - another fail # SKIP instead\n") 380 self.tap.seek(0) 381 result = subunit.TAP2SubUnit(self.tap, self.subunit) 382 self.assertEqual(0, result) 383 self.assertEqual([ 384 'test test 1 - a fail but', 385 'xfail test 1 - a fail but [', 386 'but is TODO', 387 ']', 388 'test test 2 - another fail', 389 'skip test 2 - another fail [', 390 'instead', 391 ']', 392 ], 393 self.subunit.getvalue().splitlines()) 394 395 def test_leading_comments_add_to_next_test_log(self): 396 # A file 397 # # comment 398 # ok 399 # ok 400 # results in a single test with the comment included 401 # in the first test and not the second. 402 self.tap.write("# comment\n") 403 self.tap.write("ok\n") 404 self.tap.write("ok\n") 405 self.tap.seek(0) 406 result = subunit.TAP2SubUnit(self.tap, self.subunit) 407 self.assertEqual(0, result) 408 self.assertEqual([ 409 'test test 1', 410 'success test 1 [', 411 '# comment', 412 ']', 413 'test test 2', 414 'success test 2', 415 ], 416 self.subunit.getvalue().splitlines()) 417 418 def test_trailing_comments_are_included_in_last_test_log(self): 419 # A file 420 # ok foo 421 # ok foo 422 # # comment 423 # results in a two tests, with the second having the comment 424 # attached to its log. 425 self.tap.write("ok\n") 426 self.tap.write("ok\n") 427 self.tap.write("# comment\n") 428 self.tap.seek(0) 429 result = subunit.TAP2SubUnit(self.tap, self.subunit) 430 self.assertEqual(0, result) 431 self.assertEqual([ 432 'test test 1', 433 'success test 1', 434 'test test 2', 435 'success test 2 [', 436 '# comment', 437 ']', 438 ], 439 self.subunit.getvalue().splitlines()) 440 441 442def test_suite(): 443 loader = subunit.tests.TestUtil.TestLoader() 444 result = loader.loadTestsFromName(__name__) 445 return result 446