1097a140dSpatrickimport itertools 209467b48Spatrickimport os 309467b48Spatrickfrom json import JSONEncoder 409467b48Spatrick 509467b48Spatrickfrom lit.BooleanExpression import BooleanExpression 673471bf0Spatrickfrom lit.TestTimes import read_test_times 709467b48Spatrick 809467b48Spatrick# Test result codes. 909467b48Spatrick 1009467b48Spatrickclass ResultCode(object): 1109467b48Spatrick """Test result codes.""" 1209467b48Spatrick 13097a140dSpatrick # All result codes (including user-defined ones) in declaration order 14097a140dSpatrick _all_codes = [] 15097a140dSpatrick 16097a140dSpatrick @staticmethod 17097a140dSpatrick def all_codes(): 18097a140dSpatrick return ResultCode._all_codes 19097a140dSpatrick 2009467b48Spatrick # We override __new__ and __getnewargs__ to ensure that pickling still 2109467b48Spatrick # provides unique ResultCode objects in any particular instance. 2209467b48Spatrick _instances = {} 23097a140dSpatrick 24097a140dSpatrick def __new__(cls, name, label, isFailure): 2509467b48Spatrick res = cls._instances.get(name) 2609467b48Spatrick if res is None: 2709467b48Spatrick cls._instances[name] = res = super(ResultCode, cls).__new__(cls) 2809467b48Spatrick return res 2909467b48Spatrick 30097a140dSpatrick def __getnewargs__(self): 31097a140dSpatrick return (self.name, self.label, self.isFailure) 32097a140dSpatrick 33097a140dSpatrick def __init__(self, name, label, isFailure): 3409467b48Spatrick self.name = name 35097a140dSpatrick self.label = label 3609467b48Spatrick self.isFailure = isFailure 37097a140dSpatrick ResultCode._all_codes.append(self) 3809467b48Spatrick 3909467b48Spatrick def __repr__(self): 4009467b48Spatrick return '%s%r' % (self.__class__.__name__, 4109467b48Spatrick (self.name, self.isFailure)) 4209467b48Spatrick 43097a140dSpatrick 44097a140dSpatrick# Successes 45097a140dSpatrickEXCLUDED = ResultCode('EXCLUDED', 'Excluded', False) 46097a140dSpatrickSKIPPED = ResultCode('SKIPPED', 'Skipped', False) 47097a140dSpatrickUNSUPPORTED = ResultCode('UNSUPPORTED', 'Unsupported', False) 48097a140dSpatrickPASS = ResultCode('PASS', 'Passed', False) 49097a140dSpatrickFLAKYPASS = ResultCode('FLAKYPASS', 'Passed With Retry', False) 50097a140dSpatrickXFAIL = ResultCode('XFAIL', 'Expectedly Failed', False) 51097a140dSpatrick# Failures 52097a140dSpatrickUNRESOLVED = ResultCode('UNRESOLVED', 'Unresolved', True) 53097a140dSpatrickTIMEOUT = ResultCode('TIMEOUT', 'Timed Out', True) 54097a140dSpatrickFAIL = ResultCode('FAIL', 'Failed', True) 55097a140dSpatrickXPASS = ResultCode('XPASS', 'Unexpectedly Passed', True) 56097a140dSpatrick 5709467b48Spatrick 5809467b48Spatrick# Test metric values. 5909467b48Spatrick 6009467b48Spatrickclass MetricValue(object): 6109467b48Spatrick def format(self): 6209467b48Spatrick """ 6309467b48Spatrick format() -> str 6409467b48Spatrick 6509467b48Spatrick Convert this metric to a string suitable for displaying as part of the 6609467b48Spatrick console output. 6709467b48Spatrick """ 6809467b48Spatrick raise RuntimeError("abstract method") 6909467b48Spatrick 7009467b48Spatrick def todata(self): 7109467b48Spatrick """ 7209467b48Spatrick todata() -> json-serializable data 7309467b48Spatrick 7409467b48Spatrick Convert this metric to content suitable for serializing in the JSON test 7509467b48Spatrick output. 7609467b48Spatrick """ 7709467b48Spatrick raise RuntimeError("abstract method") 7809467b48Spatrick 7909467b48Spatrickclass IntMetricValue(MetricValue): 8009467b48Spatrick def __init__(self, value): 8109467b48Spatrick self.value = value 8209467b48Spatrick 8309467b48Spatrick def format(self): 8409467b48Spatrick return str(self.value) 8509467b48Spatrick 8609467b48Spatrick def todata(self): 8709467b48Spatrick return self.value 8809467b48Spatrick 8909467b48Spatrickclass RealMetricValue(MetricValue): 9009467b48Spatrick def __init__(self, value): 9109467b48Spatrick self.value = value 9209467b48Spatrick 9309467b48Spatrick def format(self): 9409467b48Spatrick return '%.4f' % self.value 9509467b48Spatrick 9609467b48Spatrick def todata(self): 9709467b48Spatrick return self.value 9809467b48Spatrick 9909467b48Spatrickclass JSONMetricValue(MetricValue): 10009467b48Spatrick """ 10109467b48Spatrick JSONMetricValue is used for types that are representable in the output 10209467b48Spatrick but that are otherwise uninterpreted. 10309467b48Spatrick """ 10409467b48Spatrick def __init__(self, value): 10509467b48Spatrick # Ensure the value is a serializable by trying to encode it. 10609467b48Spatrick # WARNING: The value may change before it is encoded again, and may 10709467b48Spatrick # not be encodable after the change. 10809467b48Spatrick try: 10909467b48Spatrick e = JSONEncoder() 11009467b48Spatrick e.encode(value) 11109467b48Spatrick except TypeError: 11209467b48Spatrick raise 11309467b48Spatrick self.value = value 11409467b48Spatrick 11509467b48Spatrick def format(self): 11609467b48Spatrick e = JSONEncoder(indent=2, sort_keys=True) 11709467b48Spatrick return e.encode(self.value) 11809467b48Spatrick 11909467b48Spatrick def todata(self): 12009467b48Spatrick return self.value 12109467b48Spatrick 12209467b48Spatrickdef toMetricValue(value): 12309467b48Spatrick if isinstance(value, MetricValue): 12409467b48Spatrick return value 12509467b48Spatrick elif isinstance(value, int): 12609467b48Spatrick return IntMetricValue(value) 12709467b48Spatrick elif isinstance(value, float): 12809467b48Spatrick return RealMetricValue(value) 12909467b48Spatrick else: 13009467b48Spatrick # 'long' is only present in python2 13109467b48Spatrick try: 13209467b48Spatrick if isinstance(value, long): 13309467b48Spatrick return IntMetricValue(value) 13409467b48Spatrick except NameError: 13509467b48Spatrick pass 13609467b48Spatrick 13709467b48Spatrick # Try to create a JSONMetricValue and let the constructor throw 13809467b48Spatrick # if value is not a valid type. 13909467b48Spatrick return JSONMetricValue(value) 14009467b48Spatrick 14109467b48Spatrick 14209467b48Spatrick# Test results. 14309467b48Spatrick 14409467b48Spatrickclass Result(object): 14509467b48Spatrick """Wrapper for the results of executing an individual test.""" 14609467b48Spatrick 14709467b48Spatrick def __init__(self, code, output='', elapsed=None): 14809467b48Spatrick # The result code. 14909467b48Spatrick self.code = code 15009467b48Spatrick # The test output. 15109467b48Spatrick self.output = output 15209467b48Spatrick # The wall timing to execute the test, if timing. 15309467b48Spatrick self.elapsed = elapsed 15473471bf0Spatrick self.start = None 15573471bf0Spatrick self.pid = None 15609467b48Spatrick # The metrics reported by this test. 15709467b48Spatrick self.metrics = {} 15809467b48Spatrick # The micro-test results reported by this test. 15909467b48Spatrick self.microResults = {} 16009467b48Spatrick 16109467b48Spatrick def addMetric(self, name, value): 16209467b48Spatrick """ 16309467b48Spatrick addMetric(name, value) 16409467b48Spatrick 16509467b48Spatrick Attach a test metric to the test result, with the given name and list of 16609467b48Spatrick values. It is an error to attempt to attach the metrics with the same 16709467b48Spatrick name multiple times. 16809467b48Spatrick 16909467b48Spatrick Each value must be an instance of a MetricValue subclass. 17009467b48Spatrick """ 17109467b48Spatrick if name in self.metrics: 17209467b48Spatrick raise ValueError("result already includes metrics for %r" % ( 17309467b48Spatrick name,)) 17409467b48Spatrick if not isinstance(value, MetricValue): 17509467b48Spatrick raise TypeError("unexpected metric value: %r" % (value,)) 17609467b48Spatrick self.metrics[name] = value 17709467b48Spatrick 17809467b48Spatrick def addMicroResult(self, name, microResult): 17909467b48Spatrick """ 18009467b48Spatrick addMicroResult(microResult) 18109467b48Spatrick 18209467b48Spatrick Attach a micro-test result to the test result, with the given name and 18309467b48Spatrick result. It is an error to attempt to attach a micro-test with the 18409467b48Spatrick same name multiple times. 18509467b48Spatrick 18609467b48Spatrick Each micro-test result must be an instance of the Result class. 18709467b48Spatrick """ 18809467b48Spatrick if name in self.microResults: 18909467b48Spatrick raise ValueError("Result already includes microResult for %r" % ( 19009467b48Spatrick name,)) 19109467b48Spatrick if not isinstance(microResult, Result): 19209467b48Spatrick raise TypeError("unexpected MicroResult value %r" % (microResult,)) 19309467b48Spatrick self.microResults[name] = microResult 19409467b48Spatrick 19509467b48Spatrick 19609467b48Spatrick# Test classes. 19709467b48Spatrick 19809467b48Spatrickclass TestSuite: 19909467b48Spatrick """TestSuite - Information on a group of tests. 20009467b48Spatrick 20109467b48Spatrick A test suite groups together a set of logically related tests. 20209467b48Spatrick """ 20309467b48Spatrick 20409467b48Spatrick def __init__(self, name, source_root, exec_root, config): 20509467b48Spatrick self.name = name 20609467b48Spatrick self.source_root = source_root 20709467b48Spatrick self.exec_root = exec_root 20809467b48Spatrick # The test suite configuration. 20909467b48Spatrick self.config = config 21009467b48Spatrick 21173471bf0Spatrick self.test_times = read_test_times(self) 21273471bf0Spatrick 21309467b48Spatrick def getSourcePath(self, components): 21409467b48Spatrick return os.path.join(self.source_root, *components) 21509467b48Spatrick 21609467b48Spatrick def getExecPath(self, components): 21709467b48Spatrick return os.path.join(self.exec_root, *components) 21809467b48Spatrick 21909467b48Spatrickclass Test: 22009467b48Spatrick """Test - Information on a single test instance.""" 22109467b48Spatrick 222*d415bd75Srobert def __init__(self, suite, path_in_suite, config, file_path = None, gtest_json_file = None): 22309467b48Spatrick self.suite = suite 22409467b48Spatrick self.path_in_suite = path_in_suite 22509467b48Spatrick self.config = config 22609467b48Spatrick self.file_path = file_path 227*d415bd75Srobert self.gtest_json_file = gtest_json_file 22809467b48Spatrick 22909467b48Spatrick # A list of conditions under which this test is expected to fail. 230*d415bd75Srobert # Each condition is a boolean expression of features, or '*'. 231*d415bd75Srobert # These can optionally be provided by test format handlers, 232*d415bd75Srobert # and will be honored when the test result is supplied. 23309467b48Spatrick self.xfails = [] 23409467b48Spatrick 23573471bf0Spatrick # If true, ignore all items in self.xfails. 23673471bf0Spatrick self.xfail_not = False 23773471bf0Spatrick 23809467b48Spatrick # A list of conditions that must be satisfied before running the test. 23909467b48Spatrick # Each condition is a boolean expression of features. All of them 24009467b48Spatrick # must be True for the test to run. 24109467b48Spatrick self.requires = [] 24209467b48Spatrick 24309467b48Spatrick # A list of conditions that prevent execution of the test. 244*d415bd75Srobert # Each condition is a boolean expression of features. All of them 245*d415bd75Srobert # must be False for the test to run. 24609467b48Spatrick self.unsupported = [] 24709467b48Spatrick 248097a140dSpatrick # An optional number of retries allowed before the test finally succeeds. 249097a140dSpatrick # The test is run at most once plus the number of retries specified here. 250097a140dSpatrick self.allowed_retries = getattr(config, 'test_retry_attempts', 0) 251097a140dSpatrick 25209467b48Spatrick # The test result, once complete. 25309467b48Spatrick self.result = None 25409467b48Spatrick 25573471bf0Spatrick # The previous test failure state, if applicable. 25673471bf0Spatrick self.previous_failure = False 25773471bf0Spatrick 25873471bf0Spatrick # The previous test elapsed time, if applicable. 25973471bf0Spatrick self.previous_elapsed = 0.0 26073471bf0Spatrick 261*d415bd75Srobert if suite.test_times and '/'.join(path_in_suite) in suite.test_times: 26273471bf0Spatrick time = suite.test_times['/'.join(path_in_suite)] 26373471bf0Spatrick self.previous_elapsed = abs(time) 26473471bf0Spatrick self.previous_failure = time < 0 26573471bf0Spatrick 26673471bf0Spatrick 26709467b48Spatrick def setResult(self, result): 26809467b48Spatrick assert self.result is None, "result already set" 26909467b48Spatrick assert isinstance(result, Result), "unexpected result type" 270097a140dSpatrick try: 271097a140dSpatrick expected_to_fail = self.isExpectedToFail() 272097a140dSpatrick except ValueError as err: 273097a140dSpatrick # Syntax error in an XFAIL line. 274097a140dSpatrick result.code = UNRESOLVED 275097a140dSpatrick result.output = str(err) 276097a140dSpatrick else: 277097a140dSpatrick if expected_to_fail: 278097a140dSpatrick # pass -> unexpected pass 279097a140dSpatrick if result.code is PASS: 280097a140dSpatrick result.code = XPASS 281097a140dSpatrick # fail -> expected fail 282097a140dSpatrick elif result.code is FAIL: 283097a140dSpatrick result.code = XFAIL 28409467b48Spatrick self.result = result 28509467b48Spatrick 28609467b48Spatrick def isFailure(self): 28709467b48Spatrick assert self.result 28809467b48Spatrick return self.result.code.isFailure 28909467b48Spatrick 29009467b48Spatrick def getFullName(self): 29109467b48Spatrick return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) 29209467b48Spatrick 29309467b48Spatrick def getFilePath(self): 29409467b48Spatrick if self.file_path: 29509467b48Spatrick return self.file_path 29609467b48Spatrick return self.getSourcePath() 29709467b48Spatrick 29809467b48Spatrick def getSourcePath(self): 29909467b48Spatrick return self.suite.getSourcePath(self.path_in_suite) 30009467b48Spatrick 30109467b48Spatrick def getExecPath(self): 30209467b48Spatrick return self.suite.getExecPath(self.path_in_suite) 30309467b48Spatrick 30409467b48Spatrick def isExpectedToFail(self): 30509467b48Spatrick """ 30609467b48Spatrick isExpectedToFail() -> bool 30709467b48Spatrick 30809467b48Spatrick Check whether this test is expected to fail in the current 30909467b48Spatrick configuration. This check relies on the test xfails property which by 31009467b48Spatrick some test formats may not be computed until the test has first been 31109467b48Spatrick executed. 31209467b48Spatrick Throws ValueError if an XFAIL line has a syntax error. 31309467b48Spatrick """ 31409467b48Spatrick 31573471bf0Spatrick if self.xfail_not: 31673471bf0Spatrick return False 31773471bf0Spatrick 31809467b48Spatrick features = self.config.available_features 31909467b48Spatrick 320*d415bd75Srobert # Check if any of the xfails match an available feature. 32109467b48Spatrick for item in self.xfails: 32209467b48Spatrick # If this is the wildcard, it always fails. 32309467b48Spatrick if item == '*': 32409467b48Spatrick return True 32509467b48Spatrick 326*d415bd75Srobert # If this is a True expression of features, it fails. 32709467b48Spatrick try: 328*d415bd75Srobert if BooleanExpression.evaluate(item, features): 32909467b48Spatrick return True 33009467b48Spatrick except ValueError as e: 33109467b48Spatrick raise ValueError('Error in XFAIL list:\n%s' % str(e)) 33209467b48Spatrick 33309467b48Spatrick return False 33409467b48Spatrick 33509467b48Spatrick def isWithinFeatureLimits(self): 33609467b48Spatrick """ 33709467b48Spatrick isWithinFeatureLimits() -> bool 33809467b48Spatrick 33909467b48Spatrick A test is within the feature limits set by run_only_tests if 34009467b48Spatrick 1. the test's requirements ARE satisfied by the available features 34109467b48Spatrick 2. the test's requirements ARE NOT satisfied after the limiting 34209467b48Spatrick features are removed from the available features 34309467b48Spatrick 34409467b48Spatrick Throws ValueError if a REQUIRES line has a syntax error. 34509467b48Spatrick """ 34609467b48Spatrick 34709467b48Spatrick if not self.config.limit_to_features: 34809467b48Spatrick return True # No limits. Run it. 34909467b48Spatrick 35009467b48Spatrick # Check the requirements as-is (#1) 35109467b48Spatrick if self.getMissingRequiredFeatures(): 35209467b48Spatrick return False 35309467b48Spatrick 35409467b48Spatrick # Check the requirements after removing the limiting features (#2) 35509467b48Spatrick featuresMinusLimits = [f for f in self.config.available_features 35609467b48Spatrick if not f in self.config.limit_to_features] 35709467b48Spatrick if not self.getMissingRequiredFeaturesFromList(featuresMinusLimits): 35809467b48Spatrick return False 35909467b48Spatrick 36009467b48Spatrick return True 36109467b48Spatrick 36209467b48Spatrick def getMissingRequiredFeaturesFromList(self, features): 36309467b48Spatrick try: 36409467b48Spatrick return [item for item in self.requires 36509467b48Spatrick if not BooleanExpression.evaluate(item, features)] 36609467b48Spatrick except ValueError as e: 36709467b48Spatrick raise ValueError('Error in REQUIRES list:\n%s' % str(e)) 36809467b48Spatrick 36909467b48Spatrick def getMissingRequiredFeatures(self): 37009467b48Spatrick """ 37109467b48Spatrick getMissingRequiredFeatures() -> list of strings 37209467b48Spatrick 37309467b48Spatrick Returns a list of features from REQUIRES that are not satisfied." 37409467b48Spatrick Throws ValueError if a REQUIRES line has a syntax error. 37509467b48Spatrick """ 37609467b48Spatrick 37709467b48Spatrick features = self.config.available_features 37809467b48Spatrick return self.getMissingRequiredFeaturesFromList(features) 37909467b48Spatrick 38009467b48Spatrick def getUnsupportedFeatures(self): 38109467b48Spatrick """ 38209467b48Spatrick getUnsupportedFeatures() -> list of strings 38309467b48Spatrick 38409467b48Spatrick Returns a list of features from UNSUPPORTED that are present 385*d415bd75Srobert in the test configuration's features. 38609467b48Spatrick Throws ValueError if an UNSUPPORTED line has a syntax error. 38709467b48Spatrick """ 38809467b48Spatrick 38909467b48Spatrick features = self.config.available_features 39009467b48Spatrick 39109467b48Spatrick try: 39209467b48Spatrick return [item for item in self.unsupported 393*d415bd75Srobert if BooleanExpression.evaluate(item, features)] 39409467b48Spatrick except ValueError as e: 39509467b48Spatrick raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e)) 39609467b48Spatrick 397097a140dSpatrick def getUsedFeatures(self): 398097a140dSpatrick """ 399097a140dSpatrick getUsedFeatures() -> list of strings 400097a140dSpatrick 401097a140dSpatrick Returns a list of all features appearing in XFAIL, UNSUPPORTED and 402097a140dSpatrick REQUIRES annotations for this test. 403097a140dSpatrick """ 404097a140dSpatrick import lit.TestRunner 405097a140dSpatrick parsed = lit.TestRunner._parseKeywords(self.getSourcePath(), require_script=False) 406097a140dSpatrick feature_keywords = ('UNSUPPORTED:', 'REQUIRES:', 'XFAIL:') 407097a140dSpatrick boolean_expressions = itertools.chain.from_iterable( 408097a140dSpatrick parsed[k] or [] for k in feature_keywords 409097a140dSpatrick ) 410097a140dSpatrick tokens = itertools.chain.from_iterable( 411097a140dSpatrick BooleanExpression.tokenize(expr) for expr in 412097a140dSpatrick boolean_expressions if expr != '*' 413097a140dSpatrick ) 41473471bf0Spatrick matchExpressions = set(filter(BooleanExpression.isMatchExpression, tokens)) 41573471bf0Spatrick return matchExpressions 416