1#!/usr/bin/env python
2#
3# Copyright 2020 Google Inc. All Rights Reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Unit test for Google Test fail_fast.
32
33A user can specify if a Google Test program should continue test execution
34after a test failure via the GTEST_FAIL_FAST environment variable or the
35--gtest_fail_fast flag. The default value of the flag can also be changed
36by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.
37
38This script tests such functionality by invoking googletest-failfast-unittest_
39(a program written with Google Test) with different environments and command
40line flags.
41"""
42
43import os
44import gtest_test_utils
45
46# Constants.
47
48# Bazel testbridge environment variable for fail fast
49BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'
50
51# The environment variable for specifying fail fast.
52FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST'
53
54# The command line flag for specifying fail fast.
55FAIL_FAST_FLAG = 'gtest_fail_fast'
56
57# The command line flag to run disabled tests.
58RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests'
59
60# The command line flag for specifying a filter.
61FILTER_FLAG = 'gtest_filter'
62
63# Command to run the googletest-failfast-unittest_ program.
64COMMAND = gtest_test_utils.GetTestExecutablePath(
65    'googletest-failfast-unittest_')
66
67# The command line flag to tell Google Test to output the list of tests it
68# will run.
69LIST_TESTS_FLAG = '--gtest_list_tests'
70
71# Indicates whether Google Test supports death tests.
72SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(
73    [COMMAND, LIST_TESTS_FLAG]).output
74
75# Utilities.
76
77environ = os.environ.copy()
78
79
80def SetEnvVar(env_var, value):
81  """Sets the env variable to 'value'; unsets it when 'value' is None."""
82
83  if value is not None:
84    environ[env_var] = value
85  elif env_var in environ:
86    del environ[env_var]
87
88
89def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False):
90  """Runs the test program and returns its output."""
91
92  args = []
93  xml_path = os.path.join(gtest_test_utils.GetTempDir(),
94                          '.GTestFailFastUnitTest.xml')
95  args += ['--gtest_output=xml:' + xml_path]
96  if fail_fast is not None:
97    if isinstance(fail_fast, str):
98      args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)]
99    elif fail_fast:
100      args += ['--%s' % FAIL_FAST_FLAG]
101    else:
102      args += ['--no%s' % FAIL_FAST_FLAG]
103  if test_suite:
104    args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)]
105  if run_disabled:
106    args += ['--%s' % RUN_DISABLED_FLAG]
107  txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output
108  with open(xml_path) as xml_file:
109    return txt_out, xml_file.read()
110
111
112# The unit test.
113class GTestFailFastUnitTest(gtest_test_utils.TestCase):
114  """Tests the env variable or the command line flag for fail_fast."""
115
116  def testDefaultBehavior(self):
117    """Tests the behavior of not specifying the fail_fast."""
118
119    txt, _ = RunAndReturnOutput()
120    self.assertIn('22 FAILED TEST', txt)
121
122  def testGoogletestFlag(self):
123    txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True)
124    self.assertIn('1 FAILED TEST', txt)
125    self.assertIn('[  SKIPPED ] 3 tests', txt)
126
127    txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False)
128    self.assertIn('4 FAILED TEST', txt)
129    self.assertNotIn('[  SKIPPED ]', txt)
130
131  def testGoogletestEnvVar(self):
132    """Tests the behavior of specifying fail_fast via Googletest env var."""
133
134    try:
135      SetEnvVar(FAIL_FAST_ENV_VAR, '1')
136      txt, _ = RunAndReturnOutput('HasSimpleTest')
137      self.assertIn('1 FAILED TEST', txt)
138      self.assertIn('[  SKIPPED ] 3 tests', txt)
139
140      SetEnvVar(FAIL_FAST_ENV_VAR, '0')
141      txt, _ = RunAndReturnOutput('HasSimpleTest')
142      self.assertIn('4 FAILED TEST', txt)
143      self.assertNotIn('[  SKIPPED ]', txt)
144    finally:
145      SetEnvVar(FAIL_FAST_ENV_VAR, None)
146
147  def testBazelEnvVar(self):
148    """Tests the behavior of specifying fail_fast via Bazel testbridge."""
149
150    try:
151      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1')
152      txt, _ = RunAndReturnOutput('HasSimpleTest')
153      self.assertIn('1 FAILED TEST', txt)
154      self.assertIn('[  SKIPPED ] 3 tests', txt)
155
156      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
157      txt, _ = RunAndReturnOutput('HasSimpleTest')
158      self.assertIn('4 FAILED TEST', txt)
159      self.assertNotIn('[  SKIPPED ]', txt)
160    finally:
161      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
162
163  def testFlagOverridesEnvVar(self):
164    """Tests precedence of flag over env var."""
165
166    try:
167      SetEnvVar(FAIL_FAST_ENV_VAR, '0')
168      txt, _ = RunAndReturnOutput('HasSimpleTest', True)
169      self.assertIn('1 FAILED TEST', txt)
170      self.assertIn('[  SKIPPED ] 3 tests', txt)
171    finally:
172      SetEnvVar(FAIL_FAST_ENV_VAR, None)
173
174  def testGoogletestEnvVarOverridesBazelEnvVar(self):
175    """Tests that the Googletest native env var over Bazel testbridge."""
176
177    try:
178      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
179      SetEnvVar(FAIL_FAST_ENV_VAR, '1')
180      txt, _ = RunAndReturnOutput('HasSimpleTest')
181      self.assertIn('1 FAILED TEST', txt)
182      self.assertIn('[  SKIPPED ] 3 tests', txt)
183    finally:
184      SetEnvVar(FAIL_FAST_ENV_VAR, None)
185      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
186
187  def testEventListener(self):
188    txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True)
189    self.assertIn('1 FAILED TEST', txt)
190    self.assertIn('[  SKIPPED ] 3 tests', txt)
191    for expected_count, callback in [(1, 'OnTestSuiteStart'),
192                                     (5, 'OnTestStart'),
193                                     (5, 'OnTestEnd'),
194                                     (5, 'OnTestPartResult'),
195                                     (1, 'OnTestSuiteEnd')]:
196      self.assertEqual(
197          expected_count, txt.count(callback),
198          'Expected %d calls to callback %s match count on output: %s ' %
199          (expected_count, callback, txt))
200
201    txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False)
202    self.assertIn('3 FAILED TEST', txt)
203    self.assertIn('[  SKIPPED ] 1 test', txt)
204    for expected_count, callback in [(1, 'OnTestSuiteStart'),
205                                     (5, 'OnTestStart'),
206                                     (5, 'OnTestEnd'),
207                                     (5, 'OnTestPartResult'),
208                                     (1, 'OnTestSuiteEnd')]:
209      self.assertEqual(
210          expected_count, txt.count(callback),
211          'Expected %d calls to callback %s match count on output: %s ' %
212          (expected_count, callback, txt))
213
214  def assertXmlResultCount(self, result, count, xml):
215    self.assertEqual(
216        count, xml.count('result="%s"' % result),
217        'Expected \'result="%s"\' match count of %s: %s ' %
218        (result, count, xml))
219
220  def assertXmlStatusCount(self, status, count, xml):
221    self.assertEqual(
222        count, xml.count('status="%s"' % status),
223        'Expected \'status="%s"\' match count of %s: %s ' %
224        (status, count, xml))
225
226  def assertFailFastXmlAndTxtOutput(self,
227                                    fail_fast,
228                                    test_suite,
229                                    passed_count,
230                                    failure_count,
231                                    skipped_count,
232                                    suppressed_count,
233                                    run_disabled=False):
234    """Assert XML and text output of a test execution."""
235
236    txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled)
237    if failure_count > 0:
238      self.assertIn('%s FAILED TEST' % failure_count, txt)
239    if suppressed_count > 0:
240      self.assertIn('%s DISABLED TEST' % suppressed_count, txt)
241    if skipped_count > 0:
242      self.assertIn('[  SKIPPED ] %s tests' % skipped_count, txt)
243    self.assertXmlStatusCount('run',
244                              passed_count + failure_count + skipped_count, xml)
245    self.assertXmlStatusCount('notrun', suppressed_count, xml)
246    self.assertXmlResultCount('completed', passed_count + failure_count, xml)
247    self.assertXmlResultCount('skipped', skipped_count, xml)
248    self.assertXmlResultCount('suppressed', suppressed_count, xml)
249
250  def assertFailFastBehavior(self,
251                             test_suite,
252                             passed_count,
253                             failure_count,
254                             skipped_count,
255                             suppressed_count,
256                             run_disabled=False):
257    """Assert --fail_fast via flag."""
258
259    for fail_fast in ('true', '1', 't', True):
260      self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,
261                                         failure_count, skipped_count,
262                                         suppressed_count, run_disabled)
263
264  def assertNotFailFastBehavior(self,
265                                test_suite,
266                                passed_count,
267                                failure_count,
268                                skipped_count,
269                                suppressed_count,
270                                run_disabled=False):
271    """Assert --nofail_fast via flag."""
272
273    for fail_fast in ('false', '0', 'f', False):
274      self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,
275                                         failure_count, skipped_count,
276                                         suppressed_count, run_disabled)
277
278  def testFlag_HasFixtureTest(self):
279    """Tests the behavior of fail_fast and TEST_F."""
280    self.assertFailFastBehavior(
281        test_suite='HasFixtureTest',
282        passed_count=1,
283        failure_count=1,
284        skipped_count=3,
285        suppressed_count=0)
286    self.assertNotFailFastBehavior(
287        test_suite='HasFixtureTest',
288        passed_count=1,
289        failure_count=4,
290        skipped_count=0,
291        suppressed_count=0)
292
293  def testFlag_HasSimpleTest(self):
294    """Tests the behavior of fail_fast and TEST."""
295    self.assertFailFastBehavior(
296        test_suite='HasSimpleTest',
297        passed_count=1,
298        failure_count=1,
299        skipped_count=3,
300        suppressed_count=0)
301    self.assertNotFailFastBehavior(
302        test_suite='HasSimpleTest',
303        passed_count=1,
304        failure_count=4,
305        skipped_count=0,
306        suppressed_count=0)
307
308  def testFlag_HasParametersTest(self):
309    """Tests the behavior of fail_fast and TEST_P."""
310    self.assertFailFastBehavior(
311        test_suite='HasParametersSuite/HasParametersTest',
312        passed_count=0,
313        failure_count=1,
314        skipped_count=3,
315        suppressed_count=0)
316    self.assertNotFailFastBehavior(
317        test_suite='HasParametersSuite/HasParametersTest',
318        passed_count=0,
319        failure_count=4,
320        skipped_count=0,
321        suppressed_count=0)
322
323  def testFlag_HasDisabledTest(self):
324    """Tests the behavior of fail_fast and Disabled test cases."""
325    self.assertFailFastBehavior(
326        test_suite='HasDisabledTest',
327        passed_count=1,
328        failure_count=1,
329        skipped_count=2,
330        suppressed_count=1,
331        run_disabled=False)
332    self.assertNotFailFastBehavior(
333        test_suite='HasDisabledTest',
334        passed_count=1,
335        failure_count=3,
336        skipped_count=0,
337        suppressed_count=1,
338        run_disabled=False)
339
340  def testFlag_HasDisabledRunDisabledTest(self):
341    """Tests the behavior of fail_fast and Disabled test cases enabled."""
342    self.assertFailFastBehavior(
343        test_suite='HasDisabledTest',
344        passed_count=1,
345        failure_count=1,
346        skipped_count=3,
347        suppressed_count=0,
348        run_disabled=True)
349    self.assertNotFailFastBehavior(
350        test_suite='HasDisabledTest',
351        passed_count=1,
352        failure_count=4,
353        skipped_count=0,
354        suppressed_count=0,
355        run_disabled=True)
356
357  def testFlag_HasDisabledSuiteTest(self):
358    """Tests the behavior of fail_fast and Disabled test suites."""
359    self.assertFailFastBehavior(
360        test_suite='DISABLED_HasDisabledSuite',
361        passed_count=0,
362        failure_count=0,
363        skipped_count=0,
364        suppressed_count=5,
365        run_disabled=False)
366    self.assertNotFailFastBehavior(
367        test_suite='DISABLED_HasDisabledSuite',
368        passed_count=0,
369        failure_count=0,
370        skipped_count=0,
371        suppressed_count=5,
372        run_disabled=False)
373
374  def testFlag_HasDisabledSuiteRunDisabledTest(self):
375    """Tests the behavior of fail_fast and Disabled test suites enabled."""
376    self.assertFailFastBehavior(
377        test_suite='DISABLED_HasDisabledSuite',
378        passed_count=1,
379        failure_count=1,
380        skipped_count=3,
381        suppressed_count=0,
382        run_disabled=True)
383    self.assertNotFailFastBehavior(
384        test_suite='DISABLED_HasDisabledSuite',
385        passed_count=1,
386        failure_count=4,
387        skipped_count=0,
388        suppressed_count=0,
389        run_disabled=True)
390
391  if SUPPORTS_DEATH_TESTS:
392
393    def testFlag_HasDeathTest(self):
394      """Tests the behavior of fail_fast and death tests."""
395      self.assertFailFastBehavior(
396          test_suite='HasDeathTest',
397          passed_count=1,
398          failure_count=1,
399          skipped_count=3,
400          suppressed_count=0)
401      self.assertNotFailFastBehavior(
402          test_suite='HasDeathTest',
403          passed_count=1,
404          failure_count=4,
405          skipped_count=0,
406          suppressed_count=0)
407
408
409if __name__ == '__main__':
410  gtest_test_utils.Main()
411