1*d415bd75Srobertimport base64 2*d415bd75Srobertimport datetime 3097a140dSpatrickimport itertools 4097a140dSpatrickimport json 5097a140dSpatrick 6097a140dSpatrickfrom xml.sax.saxutils import quoteattr as quo 7097a140dSpatrick 8097a140dSpatrickimport lit.Test 9097a140dSpatrick 10097a140dSpatrick 11097a140dSpatrickdef by_suite_and_test_path(test): 12097a140dSpatrick # Suite names are not necessarily unique. Include object identity in sort 13097a140dSpatrick # key to avoid mixing tests of different suites. 14097a140dSpatrick return (test.suite.name, id(test.suite), test.path_in_suite) 15097a140dSpatrick 16097a140dSpatrick 17097a140dSpatrickclass JsonReport(object): 18097a140dSpatrick def __init__(self, output_file): 19097a140dSpatrick self.output_file = output_file 20097a140dSpatrick 21097a140dSpatrick def write_results(self, tests, elapsed): 22097a140dSpatrick unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} 23097a140dSpatrick tests = [t for t in tests if t.result.code not in unexecuted_codes] 24097a140dSpatrick # Construct the data we will write. 25097a140dSpatrick data = {} 26097a140dSpatrick # Encode the current lit version as a schema version. 27097a140dSpatrick data['__version__'] = lit.__versioninfo__ 28097a140dSpatrick data['elapsed'] = elapsed 29097a140dSpatrick # FIXME: Record some information on the lit configuration used? 30097a140dSpatrick # FIXME: Record information from the individual test suites? 31097a140dSpatrick 32097a140dSpatrick # Encode the tests. 33097a140dSpatrick data['tests'] = tests_data = [] 34097a140dSpatrick for test in tests: 35097a140dSpatrick test_data = { 36097a140dSpatrick 'name': test.getFullName(), 37097a140dSpatrick 'code': test.result.code.name, 38097a140dSpatrick 'output': test.result.output, 39097a140dSpatrick 'elapsed': test.result.elapsed} 40097a140dSpatrick 41097a140dSpatrick # Add test metrics, if present. 42097a140dSpatrick if test.result.metrics: 43097a140dSpatrick test_data['metrics'] = metrics_data = {} 44097a140dSpatrick for key, value in test.result.metrics.items(): 45097a140dSpatrick metrics_data[key] = value.todata() 46097a140dSpatrick 47097a140dSpatrick # Report micro-tests separately, if present 48097a140dSpatrick if test.result.microResults: 49097a140dSpatrick for key, micro_test in test.result.microResults.items(): 50097a140dSpatrick # Expand parent test name with micro test name 51097a140dSpatrick parent_name = test.getFullName() 52097a140dSpatrick micro_full_name = parent_name + ':' + key 53097a140dSpatrick 54097a140dSpatrick micro_test_data = { 55097a140dSpatrick 'name': micro_full_name, 56097a140dSpatrick 'code': micro_test.code.name, 57097a140dSpatrick 'output': micro_test.output, 58097a140dSpatrick 'elapsed': micro_test.elapsed} 59097a140dSpatrick if micro_test.metrics: 60097a140dSpatrick micro_test_data['metrics'] = micro_metrics_data = {} 61097a140dSpatrick for key, value in micro_test.metrics.items(): 62097a140dSpatrick micro_metrics_data[key] = value.todata() 63097a140dSpatrick 64097a140dSpatrick tests_data.append(micro_test_data) 65097a140dSpatrick 66097a140dSpatrick tests_data.append(test_data) 67097a140dSpatrick 68097a140dSpatrick with open(self.output_file, 'w') as file: 69097a140dSpatrick json.dump(data, file, indent=2, sort_keys=True) 70097a140dSpatrick file.write('\n') 71097a140dSpatrick 72097a140dSpatrick 7373471bf0Spatrick_invalid_xml_chars_dict = {c: None for c in range(32) if chr(c) not in ('\t', '\n', '\r')} 7473471bf0Spatrick 7573471bf0Spatrick 7673471bf0Spatrickdef remove_invalid_xml_chars(s): 7773471bf0Spatrick # According to the XML 1.0 spec, control characters other than 7873471bf0Spatrick # \t,\r, and \n are not permitted anywhere in the document 7973471bf0Spatrick # (https://www.w3.org/TR/xml/#charsets) and therefore this function 8073471bf0Spatrick # removes them to produce a valid XML document. 8173471bf0Spatrick # 8273471bf0Spatrick # Note: In XML 1.1 only \0 is illegal (https://www.w3.org/TR/xml11/#charsets) 8373471bf0Spatrick # but lit currently produces XML 1.0 output. 8473471bf0Spatrick return s.translate(_invalid_xml_chars_dict) 8573471bf0Spatrick 8673471bf0Spatrick 87097a140dSpatrickclass XunitReport(object): 88097a140dSpatrick def __init__(self, output_file): 89097a140dSpatrick self.output_file = output_file 90097a140dSpatrick self.skipped_codes = {lit.Test.EXCLUDED, 91097a140dSpatrick lit.Test.SKIPPED, lit.Test.UNSUPPORTED} 92097a140dSpatrick 93097a140dSpatrick def write_results(self, tests, elapsed): 94097a140dSpatrick tests.sort(key=by_suite_and_test_path) 95097a140dSpatrick tests_by_suite = itertools.groupby(tests, lambda t: t.suite) 96097a140dSpatrick 97097a140dSpatrick with open(self.output_file, 'w') as file: 98097a140dSpatrick file.write('<?xml version="1.0" encoding="UTF-8"?>\n') 9973471bf0Spatrick file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed)) 100097a140dSpatrick for suite, test_iter in tests_by_suite: 101097a140dSpatrick self._write_testsuite(file, suite, list(test_iter)) 102097a140dSpatrick file.write('</testsuites>\n') 103097a140dSpatrick 104097a140dSpatrick def _write_testsuite(self, file, suite, tests): 105097a140dSpatrick skipped = sum(1 for t in tests if t.result.code in self.skipped_codes) 106097a140dSpatrick failures = sum(1 for t in tests if t.isFailure()) 107097a140dSpatrick 108097a140dSpatrick name = suite.config.name.replace('.', '-') 10973471bf0Spatrick file.write(f'<testsuite name={quo(name)} tests="{len(tests)}" failures="{failures}" skipped="{skipped}">\n') 110097a140dSpatrick for test in tests: 111097a140dSpatrick self._write_test(file, test, name) 112097a140dSpatrick file.write('</testsuite>\n') 113097a140dSpatrick 114097a140dSpatrick def _write_test(self, file, test, suite_name): 115097a140dSpatrick path = '/'.join(test.path_in_suite[:-1]).replace('.', '_') 11673471bf0Spatrick class_name = f'{suite_name}.{path or suite_name}' 117097a140dSpatrick name = test.path_in_suite[-1] 118097a140dSpatrick time = test.result.elapsed or 0.0 11973471bf0Spatrick file.write(f'<testcase classname={quo(class_name)} name={quo(name)} time="{time:.2f}"') 120097a140dSpatrick 121097a140dSpatrick if test.isFailure(): 122097a140dSpatrick file.write('>\n <failure><![CDATA[') 123097a140dSpatrick # In the unlikely case that the output contains the CDATA 124097a140dSpatrick # terminator we wrap it by creating a new CDATA block. 125097a140dSpatrick output = test.result.output.replace(']]>', ']]]]><![CDATA[>') 126097a140dSpatrick if isinstance(output, bytes): 12773471bf0Spatrick output = output.decode("utf-8", 'ignore') 12873471bf0Spatrick 12973471bf0Spatrick # Failing test output sometimes contains control characters like 13073471bf0Spatrick # \x1b (e.g. if there was some -fcolor-diagnostics output) which are 13173471bf0Spatrick # not allowed inside XML files. 13273471bf0Spatrick # This causes problems with CI systems: for example, the Jenkins 13373471bf0Spatrick # JUnit XML will throw an exception when ecountering those 13473471bf0Spatrick # characters and similar problems also occur with GitLab CI. 13573471bf0Spatrick output = remove_invalid_xml_chars(output) 136097a140dSpatrick file.write(output) 137097a140dSpatrick file.write(']]></failure>\n</testcase>\n') 138097a140dSpatrick elif test.result.code in self.skipped_codes: 139097a140dSpatrick reason = self._get_skip_reason(test) 14073471bf0Spatrick file.write(f'>\n <skipped message={quo(reason)}/>\n</testcase>\n') 141097a140dSpatrick else: 142097a140dSpatrick file.write('/>\n') 143097a140dSpatrick 144097a140dSpatrick def _get_skip_reason(self, test): 145097a140dSpatrick code = test.result.code 146097a140dSpatrick if code == lit.Test.EXCLUDED: 14773471bf0Spatrick return 'Test not selected (--filter, --max-tests)' 148097a140dSpatrick if code == lit.Test.SKIPPED: 149097a140dSpatrick return 'User interrupt' 150097a140dSpatrick 151097a140dSpatrick assert code == lit.Test.UNSUPPORTED 152097a140dSpatrick features = test.getMissingRequiredFeatures() 153097a140dSpatrick if features: 154097a140dSpatrick return 'Missing required feature(s): ' + ', '.join(features) 155097a140dSpatrick return 'Unsupported configuration' 15673471bf0Spatrick 15773471bf0Spatrick 158*d415bd75Srobertdef gen_resultdb_test_entry( 159*d415bd75Srobert test_name, start_time, elapsed_time, test_output, result_code, is_expected 160*d415bd75Srobert): 161*d415bd75Srobert test_data = { 162*d415bd75Srobert 'testId': test_name, 163*d415bd75Srobert 'start_time': datetime.datetime.fromtimestamp(start_time).isoformat() + 'Z', 164*d415bd75Srobert 'duration': '%.9fs' % elapsed_time, 165*d415bd75Srobert 'summary_html': '<p><text-artifact artifact-id="artifact-content-in-request"></p>', 166*d415bd75Srobert 'artifacts': { 167*d415bd75Srobert 'artifact-content-in-request': { 168*d415bd75Srobert 'contents': base64.b64encode(test_output.encode('utf-8')).decode( 169*d415bd75Srobert 'utf-8' 170*d415bd75Srobert ), 171*d415bd75Srobert }, 172*d415bd75Srobert }, 173*d415bd75Srobert 'expected': is_expected, 174*d415bd75Srobert } 175*d415bd75Srobert if ( 176*d415bd75Srobert result_code == lit.Test.PASS 177*d415bd75Srobert or result_code == lit.Test.XPASS 178*d415bd75Srobert or result_code == lit.Test.FLAKYPASS 179*d415bd75Srobert ): 180*d415bd75Srobert test_data['status'] = 'PASS' 181*d415bd75Srobert elif result_code == lit.Test.FAIL or result_code == lit.Test.XFAIL: 182*d415bd75Srobert test_data['status'] = 'FAIL' 183*d415bd75Srobert elif ( 184*d415bd75Srobert result_code == lit.Test.UNSUPPORTED 185*d415bd75Srobert or result_code == lit.Test.SKIPPED 186*d415bd75Srobert or result_code == lit.Test.EXCLUDED 187*d415bd75Srobert ): 188*d415bd75Srobert test_data['status'] = 'SKIP' 189*d415bd75Srobert elif result_code == lit.Test.UNRESOLVED or result_code == lit.Test.TIMEOUT: 190*d415bd75Srobert test_data['status'] = 'ABORT' 191*d415bd75Srobert return test_data 192*d415bd75Srobert 193*d415bd75Srobert 194*d415bd75Srobertclass ResultDBReport(object): 195*d415bd75Srobert def __init__(self, output_file): 196*d415bd75Srobert self.output_file = output_file 197*d415bd75Srobert 198*d415bd75Srobert def write_results(self, tests, elapsed): 199*d415bd75Srobert unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} 200*d415bd75Srobert tests = [t for t in tests if t.result.code not in unexecuted_codes] 201*d415bd75Srobert data = {} 202*d415bd75Srobert data['__version__'] = lit.__versioninfo__ 203*d415bd75Srobert data['elapsed'] = elapsed 204*d415bd75Srobert # Encode the tests. 205*d415bd75Srobert data['tests'] = tests_data = [] 206*d415bd75Srobert for test in tests: 207*d415bd75Srobert tests_data.append( 208*d415bd75Srobert gen_resultdb_test_entry( 209*d415bd75Srobert test_name=test.getFullName(), 210*d415bd75Srobert start_time=test.result.start, 211*d415bd75Srobert elapsed_time=test.result.elapsed, 212*d415bd75Srobert test_output=test.result.output, 213*d415bd75Srobert result_code=test.result.code, 214*d415bd75Srobert is_expected=not test.result.code.isFailure, 215*d415bd75Srobert ) 216*d415bd75Srobert ) 217*d415bd75Srobert if test.result.microResults: 218*d415bd75Srobert for key, micro_test in test.result.microResults.items(): 219*d415bd75Srobert # Expand parent test name with micro test name 220*d415bd75Srobert parent_name = test.getFullName() 221*d415bd75Srobert micro_full_name = parent_name + ':' + key + 'microres' 222*d415bd75Srobert tests_data.append( 223*d415bd75Srobert gen_resultdb_test_entry( 224*d415bd75Srobert test_name=micro_full_name, 225*d415bd75Srobert start_time=micro_test.start 226*d415bd75Srobert if micro_test.start 227*d415bd75Srobert else test.result.start, 228*d415bd75Srobert elapsed_time=micro_test.elapsed 229*d415bd75Srobert if micro_test.elapsed 230*d415bd75Srobert else test.result.elapsed, 231*d415bd75Srobert test_output=micro_test.output, 232*d415bd75Srobert result_code=micro_test.code, 233*d415bd75Srobert is_expected=not micro_test.code.isFailure, 234*d415bd75Srobert ) 235*d415bd75Srobert ) 236*d415bd75Srobert 237*d415bd75Srobert with open(self.output_file, 'w') as file: 238*d415bd75Srobert json.dump(data, file, indent=2, sort_keys=True) 239*d415bd75Srobert file.write('\n') 240*d415bd75Srobert 241*d415bd75Srobert 24273471bf0Spatrickclass TimeTraceReport(object): 24373471bf0Spatrick def __init__(self, output_file): 24473471bf0Spatrick self.output_file = output_file 24573471bf0Spatrick self.skipped_codes = {lit.Test.EXCLUDED, 24673471bf0Spatrick lit.Test.SKIPPED, lit.Test.UNSUPPORTED} 24773471bf0Spatrick 24873471bf0Spatrick def write_results(self, tests, elapsed): 24973471bf0Spatrick # Find when first test started so we can make start times relative. 25073471bf0Spatrick first_start_time = min([t.result.start for t in tests]) 25173471bf0Spatrick events = [self._get_test_event( 25273471bf0Spatrick x, first_start_time) for x in tests if x.result.code not in self.skipped_codes] 25373471bf0Spatrick 25473471bf0Spatrick json_data = {'traceEvents': events} 25573471bf0Spatrick 25673471bf0Spatrick with open(self.output_file, "w") as time_trace_file: 25773471bf0Spatrick json.dump(json_data, time_trace_file, indent=2, sort_keys=True) 25873471bf0Spatrick 25973471bf0Spatrick def _get_test_event(self, test, first_start_time): 26073471bf0Spatrick test_name = test.getFullName() 26173471bf0Spatrick elapsed_time = test.result.elapsed or 0.0 26273471bf0Spatrick start_time = test.result.start - first_start_time if test.result.start else 0.0 26373471bf0Spatrick pid = test.result.pid or 0 26473471bf0Spatrick return { 26573471bf0Spatrick 'pid': pid, 26673471bf0Spatrick 'tid': 1, 26773471bf0Spatrick 'ph': 'X', 26873471bf0Spatrick 'ts': int(start_time * 1000000.), 26973471bf0Spatrick 'dur': int(elapsed_time * 1000000.), 27073471bf0Spatrick 'name': test_name, 27173471bf0Spatrick } 272