1#
2#  subunit: extensions to python unittest to get test results from subprocesses.
3#  Copyright (C) 2005  Robert Collins <robertc@robertcollins.net>
4#
5#  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
6#  license at the users choice. A copy of both licenses are available in the
7#  project source as Apache-2.0 and BSD. You may not use this file except in
8#  compliance with one of these two licences.
9#
10#  Unless required by applicable law or agreed to in writing, software
11#  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
13#  license you chose for the specific language governing permissions and
14#  limitations under that license.
15#
16
17"""Tests for TAP2SubUnit."""
18
19import unittest
20
21from testtools.compat import StringIO
22
23import subunit
24
25
26class TestTAP2SubUnit(unittest.TestCase):
27    """Tests for TAP2SubUnit.
28
29    These tests test TAP string data in, and subunit string data out.
30    This is ok because the subunit protocol is intended to be stable,
31    but it might be easier/pithier to write tests against TAP string in,
32    parsed subunit objects out (by hooking the subunit stream to a subunit
33    protocol server.
34    """
35
36    def setUp(self):
37        self.tap = StringIO()
38        self.subunit = StringIO()
39
40    def test_skip_entire_file(self):
41        # A file
42        # 1..- # Skipped: comment
43        # results in a single skipped test.
44        self.tap.write("1..0 # Skipped: entire file skipped\n")
45        self.tap.seek(0)
46        result = subunit.TAP2SubUnit(self.tap, self.subunit)
47        self.assertEqual(0, result)
48        self.assertEqual([
49            "test file skip",
50            "skip file skip [",
51            "Skipped: entire file skipped",
52            "]",
53            ],
54            self.subunit.getvalue().splitlines())
55
56    def test_ok_test_pass(self):
57        # A file
58        # ok
59        # results in a passed test with name 'test 1' (a synthetic name as tap
60        # does not require named fixtures - it is the first test in the tap
61        # stream).
62        self.tap.write("ok\n")
63        self.tap.seek(0)
64        result = subunit.TAP2SubUnit(self.tap, self.subunit)
65        self.assertEqual(0, result)
66        self.assertEqual([
67            "test test 1",
68            "success test 1",
69            ],
70            self.subunit.getvalue().splitlines())
71
72    def test_ok_test_number_pass(self):
73        # A file
74        # ok 1
75        # results in a passed test with name 'test 1'
76        self.tap.write("ok 1\n")
77        self.tap.seek(0)
78        result = subunit.TAP2SubUnit(self.tap, self.subunit)
79        self.assertEqual(0, result)
80        self.assertEqual([
81            "test test 1",
82            "success test 1",
83            ],
84            self.subunit.getvalue().splitlines())
85
86    def test_ok_test_number_description_pass(self):
87        # A file
88        # ok 1 - There is a description
89        # results in a passed test with name 'test 1 - There is a description'
90        self.tap.write("ok 1 - There is a description\n")
91        self.tap.seek(0)
92        result = subunit.TAP2SubUnit(self.tap, self.subunit)
93        self.assertEqual(0, result)
94        self.assertEqual([
95            "test test 1 - There is a description",
96            "success test 1 - There is a description",
97            ],
98            self.subunit.getvalue().splitlines())
99
100    def test_ok_test_description_pass(self):
101        # A file
102        # ok There is a description
103        # results in a passed test with name 'test 1 There is a description'
104        self.tap.write("ok There is a description\n")
105        self.tap.seek(0)
106        result = subunit.TAP2SubUnit(self.tap, self.subunit)
107        self.assertEqual(0, result)
108        self.assertEqual([
109            "test test 1 There is a description",
110            "success test 1 There is a description",
111            ],
112            self.subunit.getvalue().splitlines())
113
114    def test_ok_SKIP_skip(self):
115        # A file
116        # ok # SKIP
117        # results in a skkip test with name 'test 1'
118        self.tap.write("ok # SKIP\n")
119        self.tap.seek(0)
120        result = subunit.TAP2SubUnit(self.tap, self.subunit)
121        self.assertEqual(0, result)
122        self.assertEqual([
123            "test test 1",
124            "skip test 1",
125            ],
126            self.subunit.getvalue().splitlines())
127
128    def test_ok_skip_number_comment_lowercase(self):
129        self.tap.write("ok 1 # skip no samba environment available, skipping compilation\n")
130        self.tap.seek(0)
131        result = subunit.TAP2SubUnit(self.tap, self.subunit)
132        self.assertEqual(0, result)
133        self.assertEqual([
134            "test test 1",
135            "skip test 1 [",
136            "no samba environment available, skipping compilation",
137            "]"
138            ],
139            self.subunit.getvalue().splitlines())
140
141    def test_ok_number_description_SKIP_skip_comment(self):
142        # A file
143        # ok 1 foo  # SKIP Not done yet
144        # results in a skip test with name 'test 1 foo' and a log of
145        # Not done yet
146        self.tap.write("ok 1 foo  # SKIP Not done yet\n")
147        self.tap.seek(0)
148        result = subunit.TAP2SubUnit(self.tap, self.subunit)
149        self.assertEqual(0, result)
150        self.assertEqual([
151            "test test 1 foo",
152            "skip test 1 foo [",
153            "Not done yet",
154            "]",
155            ],
156            self.subunit.getvalue().splitlines())
157
158    def test_ok_SKIP_skip_comment(self):
159        # A file
160        # ok # SKIP Not done yet
161        # results in a skip test with name 'test 1' and a log of Not done yet
162        self.tap.write("ok # SKIP Not done yet\n")
163        self.tap.seek(0)
164        result = subunit.TAP2SubUnit(self.tap, self.subunit)
165        self.assertEqual(0, result)
166        self.assertEqual([
167            "test test 1",
168            "skip test 1 [",
169            "Not done yet",
170            "]",
171            ],
172            self.subunit.getvalue().splitlines())
173
174    def test_ok_TODO_xfail(self):
175        # A file
176        # ok # TODO
177        # results in a xfail test with name 'test 1'
178        self.tap.write("ok # TODO\n")
179        self.tap.seek(0)
180        result = subunit.TAP2SubUnit(self.tap, self.subunit)
181        self.assertEqual(0, result)
182        self.assertEqual([
183            "test test 1",
184            "xfail test 1",
185            ],
186            self.subunit.getvalue().splitlines())
187
188    def test_ok_TODO_xfail_comment(self):
189        # A file
190        # ok # TODO Not done yet
191        # results in a xfail test with name 'test 1' and a log of Not done yet
192        self.tap.write("ok # TODO Not done yet\n")
193        self.tap.seek(0)
194        result = subunit.TAP2SubUnit(self.tap, self.subunit)
195        self.assertEqual(0, result)
196        self.assertEqual([
197            "test test 1",
198            "xfail test 1 [",
199            "Not done yet",
200            "]",
201            ],
202            self.subunit.getvalue().splitlines())
203
204    def test_bail_out_errors(self):
205        # A file with line in it
206        # Bail out! COMMENT
207        # is treated as an error
208        self.tap.write("ok 1 foo\n")
209        self.tap.write("Bail out! Lifejacket engaged\n")
210        self.tap.seek(0)
211        result = subunit.TAP2SubUnit(self.tap, self.subunit)
212        self.assertEqual(0, result)
213        self.assertEqual([
214            "test test 1 foo",
215            "success test 1 foo",
216            "test Bail out! Lifejacket engaged",
217            "error Bail out! Lifejacket engaged",
218            ],
219            self.subunit.getvalue().splitlines())
220
221    def test_missing_test_at_end_with_plan_adds_error(self):
222        # A file
223        # 1..3
224        # ok first test
225        # not ok third test
226        # results in three tests, with the third being created
227        self.tap.write('1..3\n')
228        self.tap.write('ok first test\n')
229        self.tap.write('not ok second test\n')
230        self.tap.seek(0)
231        result = subunit.TAP2SubUnit(self.tap, self.subunit)
232        self.assertEqual(0, result)
233        self.assertEqual([
234            'test test 1 first test',
235            'success test 1 first test',
236            'test test 2 second test',
237            'failure test 2 second test',
238            'test test 3',
239            'error test 3 [',
240            'test missing from TAP output',
241            ']',
242            ],
243            self.subunit.getvalue().splitlines())
244
245    def test_missing_test_with_plan_adds_error(self):
246        # A file
247        # 1..3
248        # ok first test
249        # not ok 3 third test
250        # results in three tests, with the second being created
251        self.tap.write('1..3\n')
252        self.tap.write('ok first test\n')
253        self.tap.write('not ok 3 third test\n')
254        self.tap.seek(0)
255        result = subunit.TAP2SubUnit(self.tap, self.subunit)
256        self.assertEqual(0, result)
257        self.assertEqual([
258            'test test 1 first test',
259            'success test 1 first test',
260            'test test 2',
261            'error test 2 [',
262            'test missing from TAP output',
263            ']',
264            'test test 3 third test',
265            'failure test 3 third test',
266            ],
267            self.subunit.getvalue().splitlines())
268
269    def test_missing_test_no_plan_adds_error(self):
270        # A file
271        # ok first test
272        # not ok 3 third test
273        # results in three tests, with the second being created
274        self.tap.write('ok first test\n')
275        self.tap.write('not ok 3 third test\n')
276        self.tap.seek(0)
277        result = subunit.TAP2SubUnit(self.tap, self.subunit)
278        self.assertEqual(0, result)
279        self.assertEqual([
280            'test test 1 first test',
281            'success test 1 first test',
282            'test test 2',
283            'error test 2 [',
284            'test missing from TAP output',
285            ']',
286            'test test 3 third test',
287            'failure test 3 third test',
288            ],
289            self.subunit.getvalue().splitlines())
290
291    def test_four_tests_in_a_row_trailing_plan(self):
292        # A file
293        # ok 1 - first test in a script with no plan at all
294        # not ok 2 - second
295        # ok 3 - third
296        # not ok 4 - fourth
297        # 1..4
298        # results in four tests numbered and named
299        self.tap.write('ok 1 - first test in a script with trailing plan\n')
300        self.tap.write('not ok 2 - second\n')
301        self.tap.write('ok 3 - third\n')
302        self.tap.write('not ok 4 - fourth\n')
303        self.tap.write('1..4\n')
304        self.tap.seek(0)
305        result = subunit.TAP2SubUnit(self.tap, self.subunit)
306        self.assertEqual(0, result)
307        self.assertEqual([
308            'test test 1 - first test in a script with trailing plan',
309            'success test 1 - first test in a script with trailing plan',
310            'test test 2 - second',
311            'failure test 2 - second',
312            'test test 3 - third',
313            'success test 3 - third',
314            'test test 4 - fourth',
315            'failure test 4 - fourth'
316            ],
317            self.subunit.getvalue().splitlines())
318
319    def test_four_tests_in_a_row_with_plan(self):
320        # A file
321        # 1..4
322        # ok 1 - first test in a script with no plan at all
323        # not ok 2 - second
324        # ok 3 - third
325        # not ok 4 - fourth
326        # results in four tests numbered and named
327        self.tap.write('1..4\n')
328        self.tap.write('ok 1 - first test in a script with a plan\n')
329        self.tap.write('not ok 2 - second\n')
330        self.tap.write('ok 3 - third\n')
331        self.tap.write('not ok 4 - fourth\n')
332        self.tap.seek(0)
333        result = subunit.TAP2SubUnit(self.tap, self.subunit)
334        self.assertEqual(0, result)
335        self.assertEqual([
336            'test test 1 - first test in a script with a plan',
337            'success test 1 - first test in a script with a plan',
338            'test test 2 - second',
339            'failure test 2 - second',
340            'test test 3 - third',
341            'success test 3 - third',
342            'test test 4 - fourth',
343            'failure test 4 - fourth'
344            ],
345            self.subunit.getvalue().splitlines())
346
347    def test_four_tests_in_a_row_no_plan(self):
348        # A file
349        # ok 1 - first test in a script with no plan at all
350        # not ok 2 - second
351        # ok 3 - third
352        # not ok 4 - fourth
353        # results in four tests numbered and named
354        self.tap.write('ok 1 - first test in a script with no plan at all\n')
355        self.tap.write('not ok 2 - second\n')
356        self.tap.write('ok 3 - third\n')
357        self.tap.write('not ok 4 - fourth\n')
358        self.tap.seek(0)
359        result = subunit.TAP2SubUnit(self.tap, self.subunit)
360        self.assertEqual(0, result)
361        self.assertEqual([
362            'test test 1 - first test in a script with no plan at all',
363            'success test 1 - first test in a script with no plan at all',
364            'test test 2 - second',
365            'failure test 2 - second',
366            'test test 3 - third',
367            'success test 3 - third',
368            'test test 4 - fourth',
369            'failure test 4 - fourth'
370            ],
371            self.subunit.getvalue().splitlines())
372
373    def test_todo_and_skip(self):
374        # A file
375        # not ok 1 - a fail but # TODO but is TODO
376        # not ok 2 - another fail # SKIP instead
377        # results in two tests, numbered and commented.
378        self.tap.write("not ok 1 - a fail but # TODO but is TODO\n")
379        self.tap.write("not ok 2 - another fail # SKIP instead\n")
380        self.tap.seek(0)
381        result = subunit.TAP2SubUnit(self.tap, self.subunit)
382        self.assertEqual(0, result)
383        self.assertEqual([
384            'test test 1 - a fail but',
385            'xfail test 1 - a fail but [',
386            'but is TODO',
387            ']',
388            'test test 2 - another fail',
389            'skip test 2 - another fail [',
390            'instead',
391            ']',
392            ],
393            self.subunit.getvalue().splitlines())
394
395    def test_leading_comments_add_to_next_test_log(self):
396        # A file
397        # # comment
398        # ok
399        # ok
400        # results in a single test with the comment included
401        # in the first test and not the second.
402        self.tap.write("# comment\n")
403        self.tap.write("ok\n")
404        self.tap.write("ok\n")
405        self.tap.seek(0)
406        result = subunit.TAP2SubUnit(self.tap, self.subunit)
407        self.assertEqual(0, result)
408        self.assertEqual([
409            'test test 1',
410            'success test 1 [',
411            '# comment',
412            ']',
413            'test test 2',
414            'success test 2',
415            ],
416            self.subunit.getvalue().splitlines())
417
418    def test_trailing_comments_are_included_in_last_test_log(self):
419        # A file
420        # ok foo
421        # ok foo
422        # # comment
423        # results in a two tests, with the second having the comment
424        # attached to its log.
425        self.tap.write("ok\n")
426        self.tap.write("ok\n")
427        self.tap.write("# comment\n")
428        self.tap.seek(0)
429        result = subunit.TAP2SubUnit(self.tap, self.subunit)
430        self.assertEqual(0, result)
431        self.assertEqual([
432            'test test 1',
433            'success test 1',
434            'test test 2',
435            'success test 2 [',
436            '# comment',
437            ']',
438            ],
439            self.subunit.getvalue().splitlines())
440
441
442def test_suite():
443    loader = subunit.tests.TestUtil.TestLoader()
444    result = loader.loadTestsFromName(__name__)
445    return result
446