1#!/usr/bin/env python 2# 3# Copyright 2009 Google Inc. All Rights Reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Verifies that test shuffling works.""" 32 33import os 34from googletest.test import gtest_test_utils 35 36# Command to run the googletest-shuffle-test_ program. 37COMMAND = gtest_test_utils.GetTestExecutablePath('googletest-shuffle-test_') 38 39# The environment variables for test sharding. 40TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 41SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 42 43TEST_FILTER = 'A*.A:A*.B:C*' 44 45ALL_TESTS = [] 46ACTIVE_TESTS = [] 47FILTERED_TESTS = [] 48SHARDED_TESTS = [] 49 50SHUFFLED_ALL_TESTS = [] 51SHUFFLED_ACTIVE_TESTS = [] 52SHUFFLED_FILTERED_TESTS = [] 53SHUFFLED_SHARDED_TESTS = [] 54 55 56def AlsoRunDisabledTestsFlag(): 57 return '--gtest_also_run_disabled_tests' 58 59 60def FilterFlag(test_filter): 61 return '--gtest_filter=%s' % (test_filter,) 62 63 64def RepeatFlag(n): 65 return '--gtest_repeat=%s' % (n,) 66 67 68def ShuffleFlag(): 69 return '--gtest_shuffle' 70 71 72def RandomSeedFlag(n): 73 return '--gtest_random_seed=%s' % (n,) 74 75 76def RunAndReturnOutput(extra_env, args): 77 """Runs the test program and returns its output.""" 78 79 environ_copy = os.environ.copy() 80 environ_copy.update(extra_env) 81 82 return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output 83 84 85def GetTestsForAllIterations(extra_env, args): 86 """Runs the test program and returns a list of test lists. 87 88 Args: 89 extra_env: a map from environment variables to their values 90 args: command line flags to pass to googletest-shuffle-test_ 91 92 Returns: 93 A list where the i-th element is the list of tests run in the i-th 94 test iteration. 95 """ 96 97 test_iterations = [] 98 for line in RunAndReturnOutput(extra_env, args).split('\n'): 99 if line.startswith('----'): 100 tests = [] 101 test_iterations.append(tests) 102 elif line.strip(): 103 tests.append(line.strip()) # 'TestCaseName.TestName' 104 105 return test_iterations 106 107 108def GetTestCases(tests): 109 """Returns a list of test cases in the given full test names. 110 111 Args: 112 tests: a list of full test names 113 114 Returns: 115 A list of test cases from 'tests', in their original order. 116 Consecutive duplicates are removed. 117 """ 118 119 test_cases = [] 120 for test in tests: 121 test_case = test.split('.')[0] 122 if not test_case in test_cases: 123 test_cases.append(test_case) 124 125 return test_cases 126 127 128def CalculateTestLists(): 129 """Calculates the list of tests run under different flags.""" 130 131 if not ALL_TESTS: 132 ALL_TESTS.extend( 133 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0] 134 ) 135 136 if not ACTIVE_TESTS: 137 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 138 139 if not FILTERED_TESTS: 140 FILTERED_TESTS.extend( 141 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0] 142 ) 143 144 if not SHARDED_TESTS: 145 SHARDED_TESTS.extend( 146 GetTestsForAllIterations( 147 {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, [] 148 )[0] 149 ) 150 151 if not SHUFFLED_ALL_TESTS: 152 SHUFFLED_ALL_TESTS.extend( 153 GetTestsForAllIterations( 154 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)] 155 )[0] 156 ) 157 158 if not SHUFFLED_ACTIVE_TESTS: 159 SHUFFLED_ACTIVE_TESTS.extend( 160 GetTestsForAllIterations({}, [ShuffleFlag(), RandomSeedFlag(1)])[0] 161 ) 162 163 if not SHUFFLED_FILTERED_TESTS: 164 SHUFFLED_FILTERED_TESTS.extend( 165 GetTestsForAllIterations( 166 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)] 167 )[0] 168 ) 169 170 if not SHUFFLED_SHARDED_TESTS: 171 SHUFFLED_SHARDED_TESTS.extend( 172 GetTestsForAllIterations( 173 {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, 174 [ShuffleFlag(), RandomSeedFlag(1)], 175 )[0] 176 ) 177 178 179class GTestShuffleUnitTest(gtest_test_utils.TestCase): 180 """Tests test shuffling.""" 181 182 def setUp(self): 183 CalculateTestLists() 184 185 def testShufflePreservesNumberOfTests(self): 186 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 187 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 188 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 189 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 190 191 def testShuffleChangesTestOrder(self): 192 self.assertTrue(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 193 self.assertTrue( 194 SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS 195 ) 196 self.assertTrue( 197 SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, SHUFFLED_FILTERED_TESTS 198 ) 199 self.assertTrue( 200 SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, SHUFFLED_SHARDED_TESTS 201 ) 202 203 def testShuffleChangesTestCaseOrder(self): 204 self.assertTrue( 205 GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 206 GetTestCases(SHUFFLED_ALL_TESTS), 207 ) 208 self.assertTrue( 209 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 210 GetTestCases(SHUFFLED_ACTIVE_TESTS), 211 ) 212 self.assertTrue( 213 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 214 GetTestCases(SHUFFLED_FILTERED_TESTS), 215 ) 216 self.assertTrue( 217 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 218 GetTestCases(SHUFFLED_SHARDED_TESTS), 219 ) 220 221 def testShuffleDoesNotRepeatTest(self): 222 for test in SHUFFLED_ALL_TESTS: 223 self.assertEqual( 224 1, 225 SHUFFLED_ALL_TESTS.count(test), 226 '%s appears more than once' % (test,), 227 ) 228 for test in SHUFFLED_ACTIVE_TESTS: 229 self.assertEqual( 230 1, 231 SHUFFLED_ACTIVE_TESTS.count(test), 232 '%s appears more than once' % (test,), 233 ) 234 for test in SHUFFLED_FILTERED_TESTS: 235 self.assertEqual( 236 1, 237 SHUFFLED_FILTERED_TESTS.count(test), 238 '%s appears more than once' % (test,), 239 ) 240 for test in SHUFFLED_SHARDED_TESTS: 241 self.assertEqual( 242 1, 243 SHUFFLED_SHARDED_TESTS.count(test), 244 '%s appears more than once' % (test,), 245 ) 246 247 def testShuffleDoesNotCreateNewTest(self): 248 for test in SHUFFLED_ALL_TESTS: 249 self.assertTrue(test in ALL_TESTS, '%s is an invalid test' % (test,)) 250 for test in SHUFFLED_ACTIVE_TESTS: 251 self.assertTrue(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 252 for test in SHUFFLED_FILTERED_TESTS: 253 self.assertTrue(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 254 for test in SHUFFLED_SHARDED_TESTS: 255 self.assertTrue(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 256 257 def testShuffleIncludesAllTests(self): 258 for test in ALL_TESTS: 259 self.assertTrue(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 260 for test in ACTIVE_TESTS: 261 self.assertTrue(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 262 for test in FILTERED_TESTS: 263 self.assertTrue( 264 test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,) 265 ) 266 for test in SHARDED_TESTS: 267 self.assertTrue(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 268 269 def testShuffleLeavesDeathTestsAtFront(self): 270 non_death_test_found = False 271 for test in SHUFFLED_ACTIVE_TESTS: 272 if 'DeathTest.' in test: 273 self.assertTrue( 274 not non_death_test_found, 275 '%s appears after a non-death test' % (test,), 276 ) 277 else: 278 non_death_test_found = True 279 280 def _VerifyTestCasesDoNotInterleave(self, tests): 281 test_cases = [] 282 for test in tests: 283 [test_case, _] = test.split('.') 284 if test_cases and test_cases[-1] != test_case: 285 test_cases.append(test_case) 286 self.assertEqual( 287 1, 288 test_cases.count(test_case), 289 'Test case %s is not grouped together in %s' % (test_case, tests), 290 ) 291 292 def testShuffleDoesNotInterleaveTestCases(self): 293 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 294 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 295 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 296 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 297 298 def testShuffleRestoresOrderAfterEachIteration(self): 299 # Get the test lists in all 3 iterations, using random seed 1, 2, 300 # and 3 respectively. Google Test picks a different seed in each 301 # iteration, and this test depends on the current implementation 302 # picking successive numbers. This dependency is not ideal, but 303 # makes the test much easier to write. 304 # pylint: disable-next=unbalanced-tuple-unpacking 305 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 306 GetTestsForAllIterations( 307 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)] 308 ) 309 ) 310 311 # Make sure running the tests with random seed 1 gets the same 312 # order as in iteration 1 above. 313 tests_with_seed1 = GetTestsForAllIterations( 314 {}, [ShuffleFlag(), RandomSeedFlag(1)] 315 )[0] 316 self.assertEqual(tests_in_iteration1, tests_with_seed1) 317 318 # Make sure running the tests with random seed 2 gets the same 319 # order as in iteration 2 above. Success means that Google Test 320 # correctly restores the test order before re-shuffling at the 321 # beginning of iteration 2. 322 tests_with_seed2 = GetTestsForAllIterations( 323 {}, [ShuffleFlag(), RandomSeedFlag(2)] 324 )[0] 325 self.assertEqual(tests_in_iteration2, tests_with_seed2) 326 327 # Make sure running the tests with random seed 3 gets the same 328 # order as in iteration 3 above. Success means that Google Test 329 # correctly restores the test order before re-shuffling at the 330 # beginning of iteration 3. 331 tests_with_seed3 = GetTestsForAllIterations( 332 {}, [ShuffleFlag(), RandomSeedFlag(3)] 333 )[0] 334 self.assertEqual(tests_in_iteration3, tests_with_seed3) 335 336 def testShuffleGeneratesNewOrderInEachIteration(self): 337 # pylint: disable-next=unbalanced-tuple-unpacking 338 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 339 GetTestsForAllIterations( 340 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)] 341 ) 342 ) 343 344 self.assertTrue( 345 tests_in_iteration1 != tests_in_iteration2, tests_in_iteration1 346 ) 347 self.assertTrue( 348 tests_in_iteration1 != tests_in_iteration3, tests_in_iteration1 349 ) 350 self.assertTrue( 351 tests_in_iteration2 != tests_in_iteration3, tests_in_iteration2 352 ) 353 354 def testShuffleShardedTestsPreservesPartition(self): 355 # If we run M tests on N shards, the same M tests should be run in 356 # total, regardless of the random seeds used by the shards. 357 tests1 = GetTestsForAllIterations( 358 {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '0'}, 359 [ShuffleFlag(), RandomSeedFlag(1)], 360 )[0] 361 tests2 = GetTestsForAllIterations( 362 {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '1'}, 363 [ShuffleFlag(), RandomSeedFlag(20)], 364 )[0] 365 tests3 = GetTestsForAllIterations( 366 {TOTAL_SHARDS_ENV_VAR: '3', SHARD_INDEX_ENV_VAR: '2'}, 367 [ShuffleFlag(), RandomSeedFlag(25)], 368 )[0] 369 sorted_sharded_tests = tests1 + tests2 + tests3 370 sorted_sharded_tests.sort() 371 sorted_active_tests = [] 372 sorted_active_tests.extend(ACTIVE_TESTS) 373 sorted_active_tests.sort() 374 self.assertEqual(sorted_active_tests, sorted_sharded_tests) 375 376 377if __name__ == '__main__': 378 gtest_test_utils.Main() 379