1#!/usr/bin/env python
2#
3# Copyright 2009 Google Inc. All Rights Reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Verifies that test shuffling works."""
32
33import os
34from googletest.test import gtest_test_utils
35
36# Command to run the googletest-shuffle-test_ program.
37COMMAND = gtest_test_utils.GetTestExecutablePath('googletest-shuffle-test_')
38
39# The environment variables for test sharding.
40TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
41SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
42
43TEST_FILTER = 'A*.A:A*.B:C*'
44
45ALL_TESTS = []
46ACTIVE_TESTS = []
47FILTERED_TESTS = []
48SHARDED_TESTS = []
49
50SHUFFLED_ALL_TESTS = []
51SHUFFLED_ACTIVE_TESTS = []
52SHUFFLED_FILTERED_TESTS = []
53SHUFFLED_SHARDED_TESTS = []
54
55
56def AlsoRunDisabledTestsFlag():
57  return '--gtest_also_run_disabled_tests'
58
59
60def FilterFlag(test_filter):
61  return '--gtest_filter=%s' % (test_filter,)
62
63
64def RepeatFlag(n):
65  return '--gtest_repeat=%s' % (n,)
66
67
68def ShuffleFlag():
69  return '--gtest_shuffle'
70
71
72def RandomSeedFlag(n):
73  return '--gtest_random_seed=%s' % (n,)
74
75
76def RunAndReturnOutput(extra_env, args):
77  """Runs the test program and returns its output."""
78
79  environ_copy = os.environ.copy()
80  environ_copy.update(extra_env)
81
82  return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output
83
84
85def GetTestsForAllIterations(extra_env, args):
86  """Runs the test program and returns a list of test lists.
87
88  Args:
89    extra_env: a map from environment variables to their values
90    args: command line flags to pass to googletest-shuffle-test_
91
92  Returns:
93    A list where the i-th element is the list of tests run in the i-th
94    test iteration.
95  """
96
97  test_iterations = []
98  for line in RunAndReturnOutput(extra_env, args).split('\n'):
99    if line.startswith('----'):
100      tests = []
101      test_iterations.append(tests)
102    elif line.strip():
103      tests.append(line.strip())  # 'TestCaseName.TestName'
104
105  return test_iterations
106
107
108def GetTestCases(tests):
109  """Returns a list of test cases in the given full test names.
110
111  Args:
112    tests: a list of full test names
113
114  Returns:
115    A list of test cases from 'tests', in their original order.
116    Consecutive duplicates are removed.
117  """
118
119  test_cases = []
120  for test in tests:
121    test_case = test.split('.')[0]
122    if not test_case in test_cases:
123      test_cases.append(test_case)
124
125  return test_cases
126
127
128def CalculateTestLists():
129  """Calculates the list of tests run under different flags."""
130
131  if not ALL_TESTS:
132    ALL_TESTS.extend(
133        GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]
134    )
135
136  if not ACTIVE_TESTS:
137    ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
138
139  if not FILTERED_TESTS:
140    FILTERED_TESTS.extend(
141        GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]
142    )
143
144  if not SHARDED_TESTS:
145    SHARDED_TESTS.extend(
146        GetTestsForAllIterations(
147            {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, []
148        )[0]
149    )
150
151  if not SHUFFLED_ALL_TESTS:
152    SHUFFLED_ALL_TESTS.extend(
153        GetTestsForAllIterations(
154            {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)]
155        )[0]
156    )
157
158  if not SHUFFLED_ACTIVE_TESTS:
159    SHUFFLED_ACTIVE_TESTS.extend(
160        GetTestsForAllIterations({}, [ShuffleFlag(), RandomSeedFlag(1)])[0]
161    )
162
163  if not SHUFFLED_FILTERED_TESTS:
164    SHUFFLED_FILTERED_TESTS.extend(
165        GetTestsForAllIterations(
166            {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)]
167        )[0]
168    )
169
170  if not SHUFFLED_SHARDED_TESTS:
171    SHUFFLED_SHARDED_TESTS.extend(
172        GetTestsForAllIterations(
173            {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'},
174            [ShuffleFlag(), RandomSeedFlag(1)],
175        )[0]
176    )
177
178
179class GTestShuffleUnitTest(gtest_test_utils.TestCase):
180  """Tests test shuffling."""
181
182  def setUp(self):
183    CalculateTestLists()
184
185  def testShufflePreservesNumberOfTests(self):
186    self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
187    self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
188    self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
189    self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
190
191  def testShuffleChangesTestOrder(self):
192    self.assertTrue(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
193    self.assertTrue(
194        SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS
195    )
196    self.assertTrue(
197        SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, SHUFFLED_FILTERED_TESTS
198    )
199    self.assertTrue(
200        SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, SHUFFLED_SHARDED_TESTS
201    )
202
203  def testShuffleChangesTestCaseOrder(self):
204    self.assertTrue(
205        GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
206        GetTestCases(SHUFFLED_ALL_TESTS),
207    )
208    self.assertTrue(
209        GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
210        GetTestCases(SHUFFLED_ACTIVE_TESTS),
211    )
212    self.assertTrue(
213        GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
214        GetTestCases(SHUFFLED_FILTERED_TESTS),
215    )
216    self.assertTrue(
217        GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
218        GetTestCases(SHUFFLED_SHARDED_TESTS),
219    )
220
221  def testShuffleDoesNotRepeatTest(self):
222    for test in SHUFFLED_ALL_TESTS:
223      self.assertEqual(
224          1,
225          SHUFFLED_ALL_TESTS.count(test),
226          '%s appears more than once' % (test,),
227      )
228    for test in SHUFFLED_ACTIVE_TESTS:
229      self.assertEqual(
230          1,
231          SHUFFLED_ACTIVE_TESTS.count(test),
232          '%s appears more than once' % (test,),
233      )
234    for test in SHUFFLED_FILTERED_TESTS:
235      self.assertEqual(
236          1,
237          SHUFFLED_FILTERED_TESTS.count(test),
238          '%s appears more than once' % (test,),
239      )
240    for test in SHUFFLED_SHARDED_TESTS:
241      self.assertEqual(
242          1,
243          SHUFFLED_SHARDED_TESTS.count(test),
244          '%s appears more than once' % (test,),
245      )
246
247  def testShuffleDoesNotCreateNewTest(self):
248    for test in SHUFFLED_ALL_TESTS:
249      self.assertTrue(test in ALL_TESTS, '%s is an invalid test' % (test,))
250    for test in SHUFFLED_ACTIVE_TESTS:
251      self.assertTrue(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
252    for test in SHUFFLED_FILTERED_TESTS:
253      self.assertTrue(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
254    for test in SHUFFLED_SHARDED_TESTS:
255      self.assertTrue(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
256
257  def testShuffleIncludesAllTests(self):
258    for test in ALL_TESTS:
259      self.assertTrue(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
260    for test in ACTIVE_TESTS:
261      self.assertTrue(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
262    for test in FILTERED_TESTS:
263      self.assertTrue(
264          test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)
265      )
266    for test in SHARDED_TESTS:
267      self.assertTrue(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
268
269  def testShuffleLeavesDeathTestsAtFront(self):
270    non_death_test_found = False
271    for test in SHUFFLED_ACTIVE_TESTS:
272      if 'DeathTest.' in test:
273        self.assertTrue(
274            not non_death_test_found,
275            '%s appears after a non-death test' % (test,),
276        )
277      else:
278        non_death_test_found = True
279
280  def _VerifyTestCasesDoNotInterleave(self, tests):
281    test_cases = []
282    for test in tests:
283      [test_case, _] = test.split('.')
284      if test_cases and test_cases[-1] != test_case:
285        test_cases.append(test_case)
286        self.assertEqual(
287            1,
288            test_cases.count(test_case),
289            'Test case %s is not grouped together in %s' % (test_case, tests),
290        )
291
292  def testShuffleDoesNotInterleaveTestCases(self):
293    self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
294    self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
295    self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
296    self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
297
298  def testShuffleRestoresOrderAfterEachIteration(self):
299    # Get the test lists in all 3 iterations, using random seed 1, 2,
300    # and 3 respectively.  Google Test picks a different seed in each
301    # iteration, and this test depends on the current implementation
302    # picking successive numbers.  This dependency is not ideal, but
303    # makes the test much easier to write.
304    # pylint: disable-next=unbalanced-tuple-unpacking
305    [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
306        GetTestsForAllIterations(
307            {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]
308        )
309    )
310
311    # Make sure running the tests with random seed 1 gets the same
312    # order as in iteration 1 above.
313    tests_with_seed1 = GetTestsForAllIterations(
314        {}, [ShuffleFlag(), RandomSeedFlag(1)]
315    )[0]
316    self.assertEqual(tests_in_iteration1, tests_with_seed1)
317
318    # Make sure running the tests with random seed 2 gets the same
319    # order as in iteration 2 above.  Success means that Google Test
320    # correctly restores the test order before re-shuffling at the
321    # beginning of iteration 2.
322    tests_with_seed2 = GetTestsForAllIterations(
323        {}, [ShuffleFlag(), RandomSeedFlag(2)]
324    )[0]
325    self.assertEqual(tests_in_iteration2, tests_with_seed2)
326
327    # Make sure running the tests with random seed 3 gets the same
328    # order as in iteration 3 above.  Success means that Google Test
329    # correctly restores the test order before re-shuffling at the
330    # beginning of iteration 3.
331    tests_with_seed3 = GetTestsForAllIterations(
332        {}, [ShuffleFlag(), RandomSeedFlag(3)]
333    )[0]
334    self.assertEqual(tests_in_iteration3, tests_with_seed3)
335
336  def testShuffleGeneratesNewOrderInEachIteration(self):
337    # pylint: disable-next=unbalanced-tuple-unpacking
338    [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
339        GetTestsForAllIterations(
340            {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]
341        )
342    )
343
344    self.assertTrue(
345        tests_in_iteration1 != tests_in_iteration2, tests_in_iteration1
346    )
347    self.assertTrue(
348        tests_in_iteration1 != tests_in_iteration3, tests_in_iteration1
349    )
350    self.assertTrue(
351        tests_in_iteration2 != tests_in_iteration3, tests_in_iteration2
352    )
353
354  def testShuffleShardedTestsPreservesPartition(self):
355    # If we run M tests on N shards, the same M tests should be run in
356    # total, regardless of the random seeds used by the shards.
357    tests1 = GetTestsForAllIterations(
358        {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '0'},
359        [ShuffleFlag(), RandomSeedFlag(1)],
360    )[0]
361    tests2 = GetTestsForAllIterations(
362        {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'},
363        [ShuffleFlag(), RandomSeedFlag(20)],
364    )[0]
365    tests3 = GetTestsForAllIterations(
366        {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '2'},
367        [ShuffleFlag(), RandomSeedFlag(25)],
368    )[0]
369    sorted_sharded_tests = tests1 + tests2 + tests3
370    sorted_sharded_tests.sort()
371    sorted_active_tests = []
372    sorted_active_tests.extend(ACTIVE_TESTS)
373    sorted_active_tests.sort()
374    self.assertEqual(sorted_active_tests, sorted_sharded_tests)
375
376
377if __name__ == '__main__':
378  gtest_test_utils.Main()
379