1b89a7cc2SEnji Cooper#!/usr/bin/env python 2b89a7cc2SEnji Cooper# 3b89a7cc2SEnji Cooper# Copyright 2009 Google Inc. All Rights Reserved. 4b89a7cc2SEnji Cooper# 5b89a7cc2SEnji Cooper# Redistribution and use in source and binary forms, with or without 6b89a7cc2SEnji Cooper# modification, are permitted provided that the following conditions are 7b89a7cc2SEnji Cooper# met: 8b89a7cc2SEnji Cooper# 9b89a7cc2SEnji Cooper# * Redistributions of source code must retain the above copyright 10b89a7cc2SEnji Cooper# notice, this list of conditions and the following disclaimer. 11b89a7cc2SEnji Cooper# * Redistributions in binary form must reproduce the above 12b89a7cc2SEnji Cooper# copyright notice, this list of conditions and the following disclaimer 13b89a7cc2SEnji Cooper# in the documentation and/or other materials provided with the 14b89a7cc2SEnji Cooper# distribution. 15b89a7cc2SEnji Cooper# * Neither the name of Google Inc. nor the names of its 16b89a7cc2SEnji Cooper# contributors may be used to endorse or promote products derived from 17b89a7cc2SEnji Cooper# this software without specific prior written permission. 18b89a7cc2SEnji Cooper# 19b89a7cc2SEnji Cooper# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20b89a7cc2SEnji Cooper# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21b89a7cc2SEnji Cooper# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22b89a7cc2SEnji Cooper# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23b89a7cc2SEnji Cooper# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24b89a7cc2SEnji Cooper# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25b89a7cc2SEnji Cooper# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26b89a7cc2SEnji Cooper# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27b89a7cc2SEnji Cooper# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28b89a7cc2SEnji Cooper# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29b89a7cc2SEnji Cooper# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30b89a7cc2SEnji Cooper 31b89a7cc2SEnji Cooper"""Verifies that test shuffling works.""" 32b89a7cc2SEnji Cooper 33b89a7cc2SEnji Cooperimport os 3428f6c2f2SEnji Cooperfrom googletest.test import gtest_test_utils 35b89a7cc2SEnji Cooper 36b89a7cc2SEnji Cooper# Command to run the googletest-shuffle-test_ program. 37b89a7cc2SEnji CooperCOMMAND = gtest_test_utils.GetTestExecutablePath('googletest-shuffle-test_') 38b89a7cc2SEnji Cooper 39b89a7cc2SEnji Cooper# The environment variables for test sharding. 40b89a7cc2SEnji CooperTOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 41b89a7cc2SEnji CooperSHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 42b89a7cc2SEnji Cooper 43b89a7cc2SEnji CooperTEST_FILTER = 'A*.A:A*.B:C*' 44b89a7cc2SEnji Cooper 45b89a7cc2SEnji CooperALL_TESTS = [] 46b89a7cc2SEnji CooperACTIVE_TESTS = [] 47b89a7cc2SEnji CooperFILTERED_TESTS = [] 48b89a7cc2SEnji CooperSHARDED_TESTS = [] 49b89a7cc2SEnji Cooper 50b89a7cc2SEnji CooperSHUFFLED_ALL_TESTS = [] 51b89a7cc2SEnji CooperSHUFFLED_ACTIVE_TESTS = [] 52b89a7cc2SEnji CooperSHUFFLED_FILTERED_TESTS = [] 53b89a7cc2SEnji CooperSHUFFLED_SHARDED_TESTS = [] 54b89a7cc2SEnji Cooper 55b89a7cc2SEnji Cooper 56b89a7cc2SEnji Cooperdef AlsoRunDisabledTestsFlag(): 57b89a7cc2SEnji Cooper return '--gtest_also_run_disabled_tests' 58b89a7cc2SEnji Cooper 59b89a7cc2SEnji Cooper 60b89a7cc2SEnji Cooperdef FilterFlag(test_filter): 61b89a7cc2SEnji Cooper return '--gtest_filter=%s' % (test_filter,) 62b89a7cc2SEnji Cooper 63b89a7cc2SEnji Cooper 64b89a7cc2SEnji Cooperdef RepeatFlag(n): 65b89a7cc2SEnji Cooper return '--gtest_repeat=%s' % (n,) 66b89a7cc2SEnji Cooper 67b89a7cc2SEnji Cooper 68b89a7cc2SEnji Cooperdef ShuffleFlag(): 69b89a7cc2SEnji Cooper return '--gtest_shuffle' 70b89a7cc2SEnji Cooper 71b89a7cc2SEnji Cooper 72b89a7cc2SEnji Cooperdef RandomSeedFlag(n): 73b89a7cc2SEnji Cooper return '--gtest_random_seed=%s' % (n,) 74b89a7cc2SEnji Cooper 75b89a7cc2SEnji Cooper 76b89a7cc2SEnji Cooperdef RunAndReturnOutput(extra_env, args): 77b89a7cc2SEnji Cooper """Runs the test program and returns its output.""" 78b89a7cc2SEnji Cooper 79b89a7cc2SEnji Cooper environ_copy = os.environ.copy() 80b89a7cc2SEnji Cooper environ_copy.update(extra_env) 81b89a7cc2SEnji Cooper 82b89a7cc2SEnji Cooper return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output 83b89a7cc2SEnji Cooper 84b89a7cc2SEnji Cooper 85b89a7cc2SEnji Cooperdef GetTestsForAllIterations(extra_env, args): 86b89a7cc2SEnji Cooper """Runs the test program and returns a list of test lists. 87b89a7cc2SEnji Cooper 88b89a7cc2SEnji Cooper Args: 89b89a7cc2SEnji Cooper extra_env: a map from environment variables to their values 90b89a7cc2SEnji Cooper args: command line flags to pass to googletest-shuffle-test_ 91b89a7cc2SEnji Cooper 92b89a7cc2SEnji Cooper Returns: 93b89a7cc2SEnji Cooper A list where the i-th element is the list of tests run in the i-th 94b89a7cc2SEnji Cooper test iteration. 95b89a7cc2SEnji Cooper """ 96b89a7cc2SEnji Cooper 97b89a7cc2SEnji Cooper test_iterations = [] 98b89a7cc2SEnji Cooper for line in RunAndReturnOutput(extra_env, args).split('\n'): 99b89a7cc2SEnji Cooper if line.startswith('----'): 100b89a7cc2SEnji Cooper tests = [] 101b89a7cc2SEnji Cooper test_iterations.append(tests) 102b89a7cc2SEnji Cooper elif line.strip(): 103b89a7cc2SEnji Cooper tests.append(line.strip()) # 'TestCaseName.TestName' 104b89a7cc2SEnji Cooper 105b89a7cc2SEnji Cooper return test_iterations 106b89a7cc2SEnji Cooper 107b89a7cc2SEnji Cooper 108b89a7cc2SEnji Cooperdef GetTestCases(tests): 109b89a7cc2SEnji Cooper """Returns a list of test cases in the given full test names. 110b89a7cc2SEnji Cooper 111b89a7cc2SEnji Cooper Args: 112b89a7cc2SEnji Cooper tests: a list of full test names 113b89a7cc2SEnji Cooper 114b89a7cc2SEnji Cooper Returns: 115b89a7cc2SEnji Cooper A list of test cases from 'tests', in their original order. 116b89a7cc2SEnji Cooper Consecutive duplicates are removed. 117b89a7cc2SEnji Cooper """ 118b89a7cc2SEnji Cooper 119b89a7cc2SEnji Cooper test_cases = [] 120b89a7cc2SEnji Cooper for test in tests: 121b89a7cc2SEnji Cooper test_case = test.split('.')[0] 122b89a7cc2SEnji Cooper if not test_case in test_cases: 123b89a7cc2SEnji Cooper test_cases.append(test_case) 124b89a7cc2SEnji Cooper 125b89a7cc2SEnji Cooper return test_cases 126b89a7cc2SEnji Cooper 127b89a7cc2SEnji Cooper 128b89a7cc2SEnji Cooperdef CalculateTestLists(): 129b89a7cc2SEnji Cooper """Calculates the list of tests run under different flags.""" 130b89a7cc2SEnji Cooper 131b89a7cc2SEnji Cooper if not ALL_TESTS: 132b89a7cc2SEnji Cooper ALL_TESTS.extend( 13328f6c2f2SEnji Cooper GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0] 13428f6c2f2SEnji Cooper ) 135b89a7cc2SEnji Cooper 136b89a7cc2SEnji Cooper if not ACTIVE_TESTS: 137b89a7cc2SEnji Cooper ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 138b89a7cc2SEnji Cooper 139b89a7cc2SEnji Cooper if not FILTERED_TESTS: 140b89a7cc2SEnji Cooper FILTERED_TESTS.extend( 14128f6c2f2SEnji Cooper GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0] 14228f6c2f2SEnji Cooper ) 143b89a7cc2SEnji Cooper 144b89a7cc2SEnji Cooper if not SHARDED_TESTS: 145b89a7cc2SEnji Cooper SHARDED_TESTS.extend( 14628f6c2f2SEnji Cooper GetTestsForAllIterations( 14728f6c2f2SEnji Cooper {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, [] 14828f6c2f2SEnji Cooper )[0] 14928f6c2f2SEnji Cooper ) 150b89a7cc2SEnji Cooper 151b89a7cc2SEnji Cooper if not SHUFFLED_ALL_TESTS: 15228f6c2f2SEnji Cooper SHUFFLED_ALL_TESTS.extend( 15328f6c2f2SEnji Cooper GetTestsForAllIterations( 15428f6c2f2SEnji Cooper {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)] 15528f6c2f2SEnji Cooper )[0] 15628f6c2f2SEnji Cooper ) 157b89a7cc2SEnji Cooper 158b89a7cc2SEnji Cooper if not SHUFFLED_ACTIVE_TESTS: 15928f6c2f2SEnji Cooper SHUFFLED_ACTIVE_TESTS.extend( 16028f6c2f2SEnji Cooper GetTestsForAllIterations({}, [ShuffleFlag(), RandomSeedFlag(1)])[0] 16128f6c2f2SEnji Cooper ) 162b89a7cc2SEnji Cooper 163b89a7cc2SEnji Cooper if not SHUFFLED_FILTERED_TESTS: 16428f6c2f2SEnji Cooper SHUFFLED_FILTERED_TESTS.extend( 16528f6c2f2SEnji Cooper GetTestsForAllIterations( 16628f6c2f2SEnji Cooper {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)] 16728f6c2f2SEnji Cooper )[0] 16828f6c2f2SEnji Cooper ) 169b89a7cc2SEnji Cooper 170b89a7cc2SEnji Cooper if not SHUFFLED_SHARDED_TESTS: 171b89a7cc2SEnji Cooper SHUFFLED_SHARDED_TESTS.extend( 17228f6c2f2SEnji Cooper GetTestsForAllIterations( 17328f6c2f2SEnji Cooper {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, 17428f6c2f2SEnji Cooper [ShuffleFlag(), RandomSeedFlag(1)], 17528f6c2f2SEnji Cooper )[0] 17628f6c2f2SEnji Cooper ) 177b89a7cc2SEnji Cooper 178b89a7cc2SEnji Cooper 179b89a7cc2SEnji Cooperclass GTestShuffleUnitTest(gtest_test_utils.TestCase): 180b89a7cc2SEnji Cooper """Tests test shuffling.""" 181b89a7cc2SEnji Cooper 182b89a7cc2SEnji Cooper def setUp(self): 183b89a7cc2SEnji Cooper CalculateTestLists() 184b89a7cc2SEnji Cooper 185b89a7cc2SEnji Cooper def testShufflePreservesNumberOfTests(self): 186b89a7cc2SEnji Cooper self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 187b89a7cc2SEnji Cooper self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 188b89a7cc2SEnji Cooper self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 189b89a7cc2SEnji Cooper self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 190b89a7cc2SEnji Cooper 191b89a7cc2SEnji Cooper def testShuffleChangesTestOrder(self): 19228f6c2f2SEnji Cooper self.assertTrue(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 19328f6c2f2SEnji Cooper self.assertTrue( 19428f6c2f2SEnji Cooper SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS 19528f6c2f2SEnji Cooper ) 19628f6c2f2SEnji Cooper self.assertTrue( 19728f6c2f2SEnji Cooper SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, SHUFFLED_FILTERED_TESTS 19828f6c2f2SEnji Cooper ) 19928f6c2f2SEnji Cooper self.assertTrue( 20028f6c2f2SEnji Cooper SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, SHUFFLED_SHARDED_TESTS 20128f6c2f2SEnji Cooper ) 202b89a7cc2SEnji Cooper 203b89a7cc2SEnji Cooper def testShuffleChangesTestCaseOrder(self): 20428f6c2f2SEnji Cooper self.assertTrue( 20528f6c2f2SEnji Cooper GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 20628f6c2f2SEnji Cooper GetTestCases(SHUFFLED_ALL_TESTS), 20728f6c2f2SEnji Cooper ) 20828f6c2f2SEnji Cooper self.assertTrue( 209b89a7cc2SEnji Cooper GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 21028f6c2f2SEnji Cooper GetTestCases(SHUFFLED_ACTIVE_TESTS), 21128f6c2f2SEnji Cooper ) 21228f6c2f2SEnji Cooper self.assertTrue( 213b89a7cc2SEnji Cooper GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 21428f6c2f2SEnji Cooper GetTestCases(SHUFFLED_FILTERED_TESTS), 21528f6c2f2SEnji Cooper ) 21628f6c2f2SEnji Cooper self.assertTrue( 217b89a7cc2SEnji Cooper GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 21828f6c2f2SEnji Cooper GetTestCases(SHUFFLED_SHARDED_TESTS), 21928f6c2f2SEnji Cooper ) 220b89a7cc2SEnji Cooper 221b89a7cc2SEnji Cooper def testShuffleDoesNotRepeatTest(self): 222b89a7cc2SEnji Cooper for test in SHUFFLED_ALL_TESTS: 22328f6c2f2SEnji Cooper self.assertEqual( 22428f6c2f2SEnji Cooper 1, 22528f6c2f2SEnji Cooper SHUFFLED_ALL_TESTS.count(test), 22628f6c2f2SEnji Cooper '%s appears more than once' % (test,), 22728f6c2f2SEnji Cooper ) 228b89a7cc2SEnji Cooper for test in SHUFFLED_ACTIVE_TESTS: 22928f6c2f2SEnji Cooper self.assertEqual( 23028f6c2f2SEnji Cooper 1, 23128f6c2f2SEnji Cooper SHUFFLED_ACTIVE_TESTS.count(test), 23228f6c2f2SEnji Cooper '%s appears more than once' % (test,), 23328f6c2f2SEnji Cooper ) 234b89a7cc2SEnji Cooper for test in SHUFFLED_FILTERED_TESTS: 23528f6c2f2SEnji Cooper self.assertEqual( 23628f6c2f2SEnji Cooper 1, 23728f6c2f2SEnji Cooper SHUFFLED_FILTERED_TESTS.count(test), 23828f6c2f2SEnji Cooper '%s appears more than once' % (test,), 23928f6c2f2SEnji Cooper ) 240b89a7cc2SEnji Cooper for test in SHUFFLED_SHARDED_TESTS: 24128f6c2f2SEnji Cooper self.assertEqual( 24228f6c2f2SEnji Cooper 1, 24328f6c2f2SEnji Cooper SHUFFLED_SHARDED_TESTS.count(test), 24428f6c2f2SEnji Cooper '%s appears more than once' % (test,), 24528f6c2f2SEnji Cooper ) 246b89a7cc2SEnji Cooper 247b89a7cc2SEnji Cooper def testShuffleDoesNotCreateNewTest(self): 248b89a7cc2SEnji Cooper for test in SHUFFLED_ALL_TESTS: 24928f6c2f2SEnji Cooper self.assertTrue(test in ALL_TESTS, '%s is an invalid test' % (test,)) 250b89a7cc2SEnji Cooper for test in SHUFFLED_ACTIVE_TESTS: 25128f6c2f2SEnji Cooper self.assertTrue(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 252b89a7cc2SEnji Cooper for test in SHUFFLED_FILTERED_TESTS: 25328f6c2f2SEnji Cooper self.assertTrue(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 254b89a7cc2SEnji Cooper for test in SHUFFLED_SHARDED_TESTS: 25528f6c2f2SEnji Cooper self.assertTrue(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 256b89a7cc2SEnji Cooper 257b89a7cc2SEnji Cooper def testShuffleIncludesAllTests(self): 258b89a7cc2SEnji Cooper for test in ALL_TESTS: 25928f6c2f2SEnji Cooper self.assertTrue(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 260b89a7cc2SEnji Cooper for test in ACTIVE_TESTS: 26128f6c2f2SEnji Cooper self.assertTrue(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 262b89a7cc2SEnji Cooper for test in FILTERED_TESTS: 26328f6c2f2SEnji Cooper self.assertTrue( 26428f6c2f2SEnji Cooper test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,) 26528f6c2f2SEnji Cooper ) 266b89a7cc2SEnji Cooper for test in SHARDED_TESTS: 26728f6c2f2SEnji Cooper self.assertTrue(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 268b89a7cc2SEnji Cooper 269b89a7cc2SEnji Cooper def testShuffleLeavesDeathTestsAtFront(self): 270b89a7cc2SEnji Cooper non_death_test_found = False 271b89a7cc2SEnji Cooper for test in SHUFFLED_ACTIVE_TESTS: 272b89a7cc2SEnji Cooper if 'DeathTest.' in test: 27328f6c2f2SEnji Cooper self.assertTrue( 27428f6c2f2SEnji Cooper not non_death_test_found, 27528f6c2f2SEnji Cooper '%s appears after a non-death test' % (test,), 27628f6c2f2SEnji Cooper ) 277b89a7cc2SEnji Cooper else: 278b89a7cc2SEnji Cooper non_death_test_found = True 279b89a7cc2SEnji Cooper 280b89a7cc2SEnji Cooper def _VerifyTestCasesDoNotInterleave(self, tests): 281b89a7cc2SEnji Cooper test_cases = [] 282b89a7cc2SEnji Cooper for test in tests: 283b89a7cc2SEnji Cooper [test_case, _] = test.split('.') 284b89a7cc2SEnji Cooper if test_cases and test_cases[-1] != test_case: 285b89a7cc2SEnji Cooper test_cases.append(test_case) 28628f6c2f2SEnji Cooper self.assertEqual( 28728f6c2f2SEnji Cooper 1, 28828f6c2f2SEnji Cooper test_cases.count(test_case), 28928f6c2f2SEnji Cooper 'Test case %s is not grouped together in %s' % (test_case, tests), 29028f6c2f2SEnji Cooper ) 291b89a7cc2SEnji Cooper 292b89a7cc2SEnji Cooper def testShuffleDoesNotInterleaveTestCases(self): 293b89a7cc2SEnji Cooper self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 294b89a7cc2SEnji Cooper self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 295b89a7cc2SEnji Cooper self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 296b89a7cc2SEnji Cooper self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 297b89a7cc2SEnji Cooper 298b89a7cc2SEnji Cooper def testShuffleRestoresOrderAfterEachIteration(self): 299b89a7cc2SEnji Cooper # Get the test lists in all 3 iterations, using random seed 1, 2, 300b89a7cc2SEnji Cooper # and 3 respectively. Google Test picks a different seed in each 301b89a7cc2SEnji Cooper # iteration, and this test depends on the current implementation 302b89a7cc2SEnji Cooper # picking successive numbers. This dependency is not ideal, but 303b89a7cc2SEnji Cooper # makes the test much easier to write. 30428f6c2f2SEnji Cooper # pylint: disable-next=unbalanced-tuple-unpacking 305b89a7cc2SEnji Cooper [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 306b89a7cc2SEnji Cooper GetTestsForAllIterations( 30728f6c2f2SEnji Cooper {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)] 30828f6c2f2SEnji Cooper ) 30928f6c2f2SEnji Cooper ) 310b89a7cc2SEnji Cooper 311b89a7cc2SEnji Cooper # Make sure running the tests with random seed 1 gets the same 312b89a7cc2SEnji Cooper # order as in iteration 1 above. 31328f6c2f2SEnji Cooper tests_with_seed1 = GetTestsForAllIterations( 31428f6c2f2SEnji Cooper {}, [ShuffleFlag(), RandomSeedFlag(1)] 31528f6c2f2SEnji Cooper )[0] 316b89a7cc2SEnji Cooper self.assertEqual(tests_in_iteration1, tests_with_seed1) 317b89a7cc2SEnji Cooper 318b89a7cc2SEnji Cooper # Make sure running the tests with random seed 2 gets the same 319b89a7cc2SEnji Cooper # order as in iteration 2 above. Success means that Google Test 320b89a7cc2SEnji Cooper # correctly restores the test order before re-shuffling at the 321b89a7cc2SEnji Cooper # beginning of iteration 2. 32228f6c2f2SEnji Cooper tests_with_seed2 = GetTestsForAllIterations( 32328f6c2f2SEnji Cooper {}, [ShuffleFlag(), RandomSeedFlag(2)] 32428f6c2f2SEnji Cooper )[0] 325b89a7cc2SEnji Cooper self.assertEqual(tests_in_iteration2, tests_with_seed2) 326b89a7cc2SEnji Cooper 327b89a7cc2SEnji Cooper # Make sure running the tests with random seed 3 gets the same 328b89a7cc2SEnji Cooper # order as in iteration 3 above. Success means that Google Test 329b89a7cc2SEnji Cooper # correctly restores the test order before re-shuffling at the 330b89a7cc2SEnji Cooper # beginning of iteration 3. 33128f6c2f2SEnji Cooper tests_with_seed3 = GetTestsForAllIterations( 33228f6c2f2SEnji Cooper {}, [ShuffleFlag(), RandomSeedFlag(3)] 33328f6c2f2SEnji Cooper )[0] 334b89a7cc2SEnji Cooper self.assertEqual(tests_in_iteration3, tests_with_seed3) 335b89a7cc2SEnji Cooper 336b89a7cc2SEnji Cooper def testShuffleGeneratesNewOrderInEachIteration(self): 33728f6c2f2SEnji Cooper # pylint: disable-next=unbalanced-tuple-unpacking 338b89a7cc2SEnji Cooper [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 339b89a7cc2SEnji Cooper GetTestsForAllIterations( 34028f6c2f2SEnji Cooper {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)] 34128f6c2f2SEnji Cooper ) 34228f6c2f2SEnji Cooper ) 343b89a7cc2SEnji Cooper 34428f6c2f2SEnji Cooper self.assertTrue( 34528f6c2f2SEnji Cooper tests_in_iteration1 != tests_in_iteration2, tests_in_iteration1 34628f6c2f2SEnji Cooper ) 34728f6c2f2SEnji Cooper self.assertTrue( 34828f6c2f2SEnji Cooper tests_in_iteration1 != tests_in_iteration3, tests_in_iteration1 34928f6c2f2SEnji Cooper ) 35028f6c2f2SEnji Cooper self.assertTrue( 35128f6c2f2SEnji Cooper tests_in_iteration2 != tests_in_iteration3, tests_in_iteration2 35228f6c2f2SEnji Cooper ) 353b89a7cc2SEnji Cooper 354b89a7cc2SEnji Cooper def testShuffleShardedTestsPreservesPartition(self): 355b89a7cc2SEnji Cooper # If we run M tests on N shards, the same M tests should be run in 356b89a7cc2SEnji Cooper # total, regardless of the random seeds used by the shards. 35728f6c2f2SEnji Cooper tests1 = GetTestsForAllIterations( 35828f6c2f2SEnji Cooper {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '0'}, 35928f6c2f2SEnji Cooper [ShuffleFlag(), RandomSeedFlag(1)], 36028f6c2f2SEnji Cooper )[0] 36128f6c2f2SEnji Cooper tests2 = GetTestsForAllIterations( 36228f6c2f2SEnji Cooper {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, 36328f6c2f2SEnji Cooper [ShuffleFlag(), RandomSeedFlag(20)], 36428f6c2f2SEnji Cooper )[0] 36528f6c2f2SEnji Cooper tests3 = GetTestsForAllIterations( 36628f6c2f2SEnji Cooper {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '2'}, 36728f6c2f2SEnji Cooper [ShuffleFlag(), RandomSeedFlag(25)], 36828f6c2f2SEnji Cooper )[0] 369b89a7cc2SEnji Cooper sorted_sharded_tests = tests1 + tests2 + tests3 370b89a7cc2SEnji Cooper sorted_sharded_tests.sort() 371b89a7cc2SEnji Cooper sorted_active_tests = [] 372b89a7cc2SEnji Cooper sorted_active_tests.extend(ACTIVE_TESTS) 373b89a7cc2SEnji Cooper sorted_active_tests.sort() 374b89a7cc2SEnji Cooper self.assertEqual(sorted_active_tests, sorted_sharded_tests) 375b89a7cc2SEnji Cooper 37628f6c2f2SEnji Cooper 377b89a7cc2SEnji Cooperif __name__ == '__main__': 378b89a7cc2SEnji Cooper gtest_test_utils.Main() 379