109467b48Spatrickimport multiprocessing 209467b48Spatrickimport os 309467b48Spatrickimport time 409467b48Spatrick 509467b48Spatrickimport lit.Test 609467b48Spatrickimport lit.util 709467b48Spatrickimport lit.worker 809467b48Spatrick 909467b48Spatrick 10097a140dSpatrickclass MaxFailuresError(Exception): 11097a140dSpatrick pass 12097a140dSpatrickclass TimeoutError(Exception): 13097a140dSpatrick pass 1409467b48Spatrick 1509467b48Spatrick 1609467b48Spatrickclass Run(object): 1709467b48Spatrick """A concrete, configured testing run.""" 1809467b48Spatrick 19097a140dSpatrick def __init__(self, tests, lit_config, workers, progress_callback, 20097a140dSpatrick max_failures, timeout): 2109467b48Spatrick self.tests = tests 2209467b48Spatrick self.lit_config = lit_config 23097a140dSpatrick self.workers = workers 2409467b48Spatrick self.progress_callback = progress_callback 2509467b48Spatrick self.max_failures = max_failures 2609467b48Spatrick self.timeout = timeout 27097a140dSpatrick assert workers > 0 2809467b48Spatrick 2909467b48Spatrick def execute(self): 3009467b48Spatrick """ 3109467b48Spatrick Execute the tests in the run using up to the specified number of 3209467b48Spatrick parallel tasks, and inform the caller of each individual result. The 3309467b48Spatrick provided tests should be a subset of the tests available in this run 3409467b48Spatrick object. 3509467b48Spatrick 3609467b48Spatrick The progress_callback will be invoked for each completed test. 3709467b48Spatrick 3809467b48Spatrick If timeout is non-None, it should be a time in seconds after which to 3909467b48Spatrick stop executing tests. 4009467b48Spatrick 4109467b48Spatrick Returns the elapsed testing time. 4209467b48Spatrick 4309467b48Spatrick Upon completion, each test in the run will have its result 4409467b48Spatrick computed. Tests which were not actually executed (for any reason) will 45097a140dSpatrick be marked SKIPPED. 4609467b48Spatrick """ 47097a140dSpatrick self.failures = 0 4809467b48Spatrick 4909467b48Spatrick # Larger timeouts (one year, positive infinity) don't work on Windows. 5009467b48Spatrick one_week = 7 * 24 * 60 * 60 # days * hours * minutes * seconds 5109467b48Spatrick timeout = self.timeout or one_week 5209467b48Spatrick deadline = time.time() + timeout 5309467b48Spatrick 54097a140dSpatrick try: 5509467b48Spatrick self._execute(deadline) 56097a140dSpatrick finally: 57097a140dSpatrick skipped = lit.Test.Result(lit.Test.SKIPPED) 5809467b48Spatrick for test in self.tests: 5909467b48Spatrick if test.result is None: 60097a140dSpatrick test.setResult(skipped) 6109467b48Spatrick 6209467b48Spatrick def _execute(self, deadline): 6309467b48Spatrick self._increase_process_limit() 6409467b48Spatrick 65097a140dSpatrick semaphores = {k: multiprocessing.BoundedSemaphore(v) 66097a140dSpatrick for k, v in self.lit_config.parallelism_groups.items() 67097a140dSpatrick if v is not None} 68097a140dSpatrick 6909467b48Spatrick pool = multiprocessing.Pool(self.workers, lit.worker.initialize, 7009467b48Spatrick (self.lit_config, semaphores)) 7109467b48Spatrick 7209467b48Spatrick async_results = [ 7309467b48Spatrick pool.apply_async(lit.worker.execute, args=[test], 74097a140dSpatrick callback=self.progress_callback) 7509467b48Spatrick for test in self.tests] 7609467b48Spatrick pool.close() 7709467b48Spatrick 7809467b48Spatrick try: 79097a140dSpatrick self._wait_for(async_results, deadline) 80097a140dSpatrick except: 8109467b48Spatrick pool.terminate() 82097a140dSpatrick raise 83097a140dSpatrick finally: 8409467b48Spatrick pool.join() 8509467b48Spatrick 86097a140dSpatrick def _wait_for(self, async_results, deadline): 87097a140dSpatrick timeout = deadline - time.time() 88097a140dSpatrick for idx, ar in enumerate(async_results): 89097a140dSpatrick try: 90097a140dSpatrick test = ar.get(timeout) 91097a140dSpatrick except multiprocessing.TimeoutError: 92097a140dSpatrick raise TimeoutError() 93097a140dSpatrick else: 94097a140dSpatrick self._update_test(self.tests[idx], test) 95097a140dSpatrick if test.isFailure(): 96097a140dSpatrick self.failures += 1 97097a140dSpatrick if self.failures == self.max_failures: 98097a140dSpatrick raise MaxFailuresError() 99097a140dSpatrick 100097a140dSpatrick # Update local test object "in place" from remote test object. This 101097a140dSpatrick # ensures that the original test object which is used for printing test 102097a140dSpatrick # results reflects the changes. 103097a140dSpatrick def _update_test(self, local_test, remote_test): 104097a140dSpatrick # Needed for getMissingRequiredFeatures() 105097a140dSpatrick local_test.requires = remote_test.requires 106097a140dSpatrick local_test.result = remote_test.result 107097a140dSpatrick 10809467b48Spatrick # TODO(yln): interferes with progress bar 10909467b48Spatrick # Some tests use threads internally, and at least on Linux each of these 11009467b48Spatrick # threads counts toward the current process limit. Try to raise the (soft) 11109467b48Spatrick # process limit so that tests don't fail due to resource exhaustion. 11209467b48Spatrick def _increase_process_limit(self): 113*73471bf0Spatrick ncpus = lit.util.usable_core_count() 11409467b48Spatrick desired_limit = self.workers * ncpus * 2 # the 2 is a safety factor 11509467b48Spatrick 11609467b48Spatrick # Importing the resource module will likely fail on Windows. 11709467b48Spatrick try: 11809467b48Spatrick import resource 11909467b48Spatrick NPROC = resource.RLIMIT_NPROC 12009467b48Spatrick 12109467b48Spatrick soft_limit, hard_limit = resource.getrlimit(NPROC) 12209467b48Spatrick desired_limit = min(desired_limit, hard_limit) 12309467b48Spatrick 12409467b48Spatrick if soft_limit < desired_limit: 12509467b48Spatrick resource.setrlimit(NPROC, (desired_limit, hard_limit)) 12609467b48Spatrick self.lit_config.note('Raised process limit from %d to %d' % \ 12709467b48Spatrick (soft_limit, desired_limit)) 12809467b48Spatrick except Exception as ex: 12909467b48Spatrick # Warn, unless this is Windows, in which case this is expected. 13009467b48Spatrick if os.name != 'nt': 13109467b48Spatrick self.lit_config.warning('Failed to raise process limit: %s' % ex) 132