1#!/usr/bin/env python 2# 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6########################################################################## 7# 8# This is a collection of helper tools to get stuff done in NSS. 9# 10 11import sys 12import argparse 13import fnmatch 14import io 15import subprocess 16import os 17import platform 18import shutil 19import tarfile 20import tempfile 21 22from hashlib import sha256 23 24DEVNULL = open(os.devnull, 'wb') 25cwd = os.path.dirname(os.path.abspath(__file__)) 26 27def run_tests(test, cycles="standard", env={}, silent=False): 28 domsuf = os.getenv('DOMSUF', "localdomain") 29 host = os.getenv('HOST', "localhost") 30 env = env.copy() 31 env.update({ 32 "NSS_TESTS": test, 33 "NSS_CYCLES": cycles, 34 "DOMSUF": domsuf, 35 "HOST": host 36 }) 37 os_env = os.environ 38 os_env.update(env) 39 command = cwd + "/tests/all.sh" 40 stdout = stderr = DEVNULL if silent else None 41 subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr) 42 43 44class coverityAction(argparse.Action): 45 46 def get_coverity_remote_cfg(self): 47 secret_name = 'project/relman/coverity-nss' 48 secrets_url = 'http://taskcluster/secrets/v1/secret/{}'.format(secret_name) 49 50 print('Using symbol upload token from the secrets service: "{}"'. 51 format(secrets_url)) 52 53 import requests 54 res = requests.get(secrets_url) 55 res.raise_for_status() 56 secret = res.json() 57 cov_config = secret['secret'] if 'secret' in secret else None 58 59 if cov_config is None: 60 print('Ill formatted secret for Coverity. Aborting analysis.') 61 return None 62 63 return cov_config 64 65 def get_coverity_local_cfg(self, path): 66 try: 67 import yaml 68 file_handler = open(path) 69 config = yaml.safe_load(file_handler) 70 except Exception: 71 print('Unable to load coverity config from {}'.format(path)) 72 return None 73 return config 74 75 def get_cov_config(self, path): 76 cov_config = None 77 if self.local_config: 78 cov_config = self.get_coverity_local_cfg(path) 79 else: 80 cov_config = self.get_coverity_remote_cfg() 81 82 if cov_config is None: 83 print('Unable to load Coverity config.') 84 return 1 85 86 self.cov_analysis_url = cov_config.get('package_url') 87 self.cov_package_name = cov_config.get('package_name') 88 self.cov_url = cov_config.get('server_url') 89 self.cov_port = cov_config.get('server_port') 90 self.cov_auth = cov_config.get('auth_key') 91 self.cov_package_ver = cov_config.get('package_ver') 92 self.cov_full_stack = cov_config.get('full_stack', False) 93 94 return 0 95 96 def download_coverity(self): 97 if self.cov_url is None or self.cov_port is None or self.cov_analysis_url is None or self.cov_auth is None: 98 print('Missing Coverity config options!') 99 return 1 100 101 COVERITY_CONFIG = ''' 102 { 103 "type": "Coverity configuration", 104 "format_version": 1, 105 "settings": { 106 "server": { 107 "host": "%s", 108 "port": %s, 109 "ssl" : true, 110 "on_new_cert" : "trust", 111 "auth_key_file": "%s" 112 }, 113 "stream": "NSS", 114 "cov_run_desktop": { 115 "build_cmd": ["%s"], 116 "clean_cmd": ["%s", "-cc"], 117 } 118 } 119 } 120 ''' 121 # Generate the coverity.conf and auth files 122 build_cmd = os.path.join(cwd, 'build.sh') 123 cov_auth_path = os.path.join(self.cov_state_path, 'auth') 124 cov_setup_path = os.path.join(self.cov_state_path, 'coverity.conf') 125 cov_conf = COVERITY_CONFIG % (self.cov_url, self.cov_port, cov_auth_path, build_cmd, build_cmd) 126 127 def download(artifact_url, target): 128 import requests 129 resp = requests.get(artifact_url, verify=False, stream=True) 130 resp.raise_for_status() 131 132 # Extract archive into destination 133 with tarfile.open(fileobj=io.BytesIO(resp.content)) as tar: 134 tar.extractall(target) 135 136 download(self.cov_analysis_url, self.cov_state_path) 137 138 with open(cov_auth_path, 'w') as f: 139 f.write(self.cov_auth) 140 141 # Modify it's permission to 600 142 os.chmod(cov_auth_path, 0o600) 143 144 with open(cov_setup_path, 'a') as f: 145 f.write(cov_conf) 146 147 def setup_coverity(self, config_path, storage_path=None, force_download=True): 148 rc = self.get_cov_config(config_path) 149 150 if rc != 0: 151 return rc 152 153 if storage_path is None: 154 # If storage_path is None we set the context of the coverity into the cwd. 155 storage_path = cwd 156 157 self.cov_state_path = os.path.join(storage_path, "coverity") 158 159 if force_download is True or not os.path.exists(self.cov_state_path): 160 shutil.rmtree(self.cov_state_path, ignore_errors=True) 161 os.mkdir(self.cov_state_path) 162 163 # Download everything that we need for Coverity from out private instance 164 self.download_coverity() 165 166 self.cov_path = os.path.join(self.cov_state_path, self.cov_package_name) 167 self.cov_run_desktop = os.path.join(self.cov_path, 'bin', 'cov-run-desktop') 168 self.cov_translate = os.path.join(self.cov_path, 'bin', 'cov-translate') 169 self.cov_configure = os.path.join(self.cov_path, 'bin', 'cov-configure') 170 self.cov_work_path = os.path.join(self.cov_state_path, 'data-coverity') 171 self.cov_idir_path = os.path.join(self.cov_work_path, self.cov_package_ver, 'idir') 172 173 if not os.path.exists(self.cov_path) or \ 174 not os.path.exists(self.cov_run_desktop) or \ 175 not os.path.exists(self.cov_translate) or \ 176 not os.path.exists(self.cov_configure): 177 print('Missing Coverity in {}'.format(self.cov_path)) 178 return 1 179 180 return 0 181 182 def run_process(self, args, cwd=cwd): 183 proc = subprocess.Popen(args, cwd=cwd) 184 status = None 185 while status is None: 186 try: 187 status = proc.wait() 188 except KeyboardInterrupt: 189 pass 190 return status 191 192 def cov_is_file_in_source(self, abs_path): 193 if os.path.islink(abs_path): 194 abs_path = os.path.realpath(abs_path) 195 return abs_path 196 197 def dump_cov_artifact(self, cov_results, source, output): 198 import json 199 200 def relpath(path): 201 '''Build path relative to repository root''' 202 if path.startswith(cwd): 203 return os.path.relpath(path, cwd) 204 return path 205 206 # Parse Coverity json into structured issues 207 with open(cov_results) as f: 208 result = json.load(f) 209 210 # Parse the issues to a standard json format 211 issues_dict = {'files': {}} 212 213 files_list = issues_dict['files'] 214 215 def build_element(issue): 216 # We look only for main event 217 event_path = next((event for event in issue['events'] if event['main'] is True), None) 218 219 dict_issue = { 220 'line': issue['mainEventLineNumber'], 221 'flag': issue['checkerName'], 222 'message': event_path['eventDescription'], 223 'extra': { 224 'category': issue['checkerProperties']['category'], 225 'stateOnServer': issue['stateOnServer'], 226 'stack': [] 227 } 228 } 229 230 # Embed all events into extra message 231 for event in issue['events']: 232 dict_issue['extra']['stack'].append({'file_path': relpath(event['strippedFilePathname']), 233 'line_number': event['lineNumber'], 234 'path_type': event['eventTag'], 235 'description': event['eventDescription']}) 236 237 return dict_issue 238 239 for issue in result['issues']: 240 path = self.cov_is_file_in_source(issue['strippedMainEventFilePathname']) 241 if path is None: 242 # Since we skip a result we should log it 243 print('Skipping CID: {0} from file: {1} since it\'s not related with the current patch.'.format( 244 issue['stateOnServer']['cid'], issue['strippedMainEventFilePathname'])) 245 continue 246 # If path does not start with `cwd` skip it 247 if not path.startswith(cwd): 248 continue 249 path = relpath(path) 250 if path in files_list: 251 files_list[path]['warnings'].append(build_element(issue)) 252 else: 253 files_list[path] = {'warnings': [build_element(issue)]} 254 255 with open(output, 'w') as f: 256 json.dump(issues_dict, f) 257 258 def mutate_paths(self, paths): 259 for index in xrange(len(paths)): 260 paths[index] = os.path.abspath(paths[index]) 261 262 def __call__(self, parser, args, paths, option_string=None): 263 self.local_config = True 264 config_path = args.config 265 storage_path = args.storage 266 267 have_paths = True 268 if len(paths) == 0: 269 have_paths = False 270 print('No files have been specified for analysis, running Coverity on the entire project.') 271 272 self.mutate_paths(paths) 273 274 if config_path is None: 275 self.local_config = False 276 print('No coverity config path has been specified, so running in automation.') 277 if 'NSS_AUTOMATION' not in os.environ: 278 print('Coverity based static-analysis cannot be ran outside automation.') 279 return 1 280 281 rc = self.setup_coverity(config_path, storage_path, args.force) 282 if rc != 0: 283 return 1 284 285 # First run cov-run-desktop --setup in order to setup the analysis env 286 cmd = [self.cov_run_desktop, '--setup'] 287 print('Running {} --setup'.format(self.cov_run_desktop)) 288 289 rc = self.run_process(args=cmd, cwd=self.cov_path) 290 291 if rc != 0: 292 print('Running {} --setup failed!'.format(self.cov_run_desktop)) 293 return rc 294 295 cov_result = os.path.join(self.cov_state_path, 'cov-results.json') 296 297 # Once the capture is performed we need to do the actual Coverity Desktop analysis 298 if have_paths: 299 cmd = [self.cov_run_desktop, '--json-output-v6', cov_result] + paths 300 else: 301 cmd = [self.cov_run_desktop, '--json-output-v6', cov_result, '--analyze-captured-source'] 302 303 print('Running Coverity Analysis for {}'.format(cmd)) 304 305 rc = self.run_process(cmd, cwd=self.cov_state_path) 306 307 if rc != 0: 308 print('Coverity Analysis failed!') 309 310 # On automation, like try, we want to build an artifact with the results. 311 if 'NSS_AUTOMATION' in os.environ: 312 self.dump_cov_artifact(cov_result, cov_result, "/home/worker/nss/coverity/coverity.json") 313 314 315class cfAction(argparse.Action): 316 docker_command = None 317 restorecon = None 318 319 def __call__(self, parser, args, values, option_string=None): 320 self.setDockerCommand(args) 321 322 if values: 323 files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values] 324 else: 325 files = self.modifiedFiles() 326 327 # First check if we can run docker. 328 try: 329 with open(os.devnull, "w") as f: 330 subprocess.check_call( 331 self.docker_command + ["images"], stdout=f) 332 except: 333 self.docker_command = None 334 335 if self.docker_command is None: 336 print("warning: running clang-format directly, which isn't guaranteed to be correct") 337 command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files 338 repr(command) 339 subprocess.call(command) 340 return 341 342 files = [os.path.join('/home/worker/nss', x) for x in files] 343 docker_image = 'clang-format-service:latest' 344 cf_docker_folder = cwd + "/automation/clang-format" 345 346 # Build the image if necessary. 347 if self.filesChanged(cf_docker_folder): 348 self.buildImage(docker_image, cf_docker_folder) 349 350 # Check if we have the docker image. 351 try: 352 command = self.docker_command + [ 353 "image", "inspect", "clang-format-service:latest" 354 ] 355 with open(os.devnull, "w") as f: 356 subprocess.check_call(command, stdout=f) 357 except: 358 print("I have to build the docker image first.") 359 self.buildImage(docker_image, cf_docker_folder) 360 361 command = self.docker_command + [ 362 'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image 363 ] 364 # The clang format script returns 1 if something's to do. We don't 365 # care. 366 subprocess.call(command + files) 367 if self.restorecon is not None: 368 subprocess.call([self.restorecon, '-R', cwd]) 369 370 def filesChanged(self, path): 371 hash = sha256() 372 for dirname, dirnames, files in os.walk(path): 373 for file in files: 374 with open(os.path.join(dirname, file), "rb") as f: 375 hash.update(f.read()) 376 chk_file = cwd + "/.chk" 377 old_chk = "" 378 new_chk = hash.hexdigest() 379 if os.path.exists(chk_file): 380 with open(chk_file) as f: 381 old_chk = f.readline() 382 if old_chk != new_chk: 383 with open(chk_file, "w+") as f: 384 f.write(new_chk) 385 return True 386 return False 387 388 def buildImage(self, docker_image, cf_docker_folder): 389 command = self.docker_command + [ 390 "build", "-t", docker_image, cf_docker_folder 391 ] 392 subprocess.check_call(command) 393 return 394 395 def setDockerCommand(self, args): 396 from distutils.spawn import find_executable 397 if platform.system() == "Linux": 398 self.restorecon = find_executable("restorecon") 399 dcmd = find_executable("docker") 400 if dcmd is not None: 401 self.docker_command = [dcmd] 402 if not args.noroot: 403 self.docker_command = ["sudo"] + self.docker_command 404 else: 405 self.docker_command = None 406 407 def modifiedFiles(self): 408 files = [] 409 if os.path.exists(os.path.join(cwd, '.hg')): 410 st = subprocess.Popen(['hg', 'status', '-m', '-a'], 411 cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True) 412 for line in iter(st.stdout.readline, ''): 413 files += [line[2:].rstrip()] 414 elif os.path.exists(os.path.join(cwd, '.git')): 415 st = subprocess.Popen(['git', 'status', '--porcelain'], 416 cwd=cwd, stdout=subprocess.PIPE) 417 for line in iter(st.stdout.readline, ''): 418 if line[1] == 'M' or line[1] != 'D' and \ 419 (line[0] == 'M' or line[0] == 'A' or 420 line[0] == 'C' or line[0] == 'U'): 421 files += [line[3:].rstrip()] 422 elif line[0] == 'R': 423 files += [line[line.index(' -> ', beg=4) + 4:]] 424 else: 425 print('Warning: neither mercurial nor git detected!') 426 427 def isFormatted(x): 428 return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h' 429 return [x for x in files if isFormatted(x)] 430 431 432class buildAction(argparse.Action): 433 434 def __call__(self, parser, args, values, option_string=None): 435 subprocess.check_call([cwd + "/build.sh"] + values) 436 437 438class testAction(argparse.Action): 439 440 def __call__(self, parser, args, values, option_string=None): 441 run_tests(values) 442 443 444class covAction(argparse.Action): 445 446 def runSslGtests(self, outdir): 447 env = { 448 "GTESTFILTER": "*", # Prevent parallel test runs. 449 "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir, 450 "NSS_DEFAULT_DB_TYPE": "dbm" 451 } 452 453 run_tests("ssl_gtests", env=env, silent=True) 454 455 def findSanCovFile(self, outdir): 456 for file in os.listdir(outdir): 457 if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'): 458 return os.path.join(outdir, file) 459 460 return None 461 462 def __call__(self, parser, args, values, option_string=None): 463 outdir = args.outdir 464 print("Output directory: " + outdir) 465 466 print("\nBuild with coverage sanitizers...\n") 467 sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp" 468 subprocess.check_call([ 469 os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db", 470 "--sancov=" + sancov_args 471 ]) 472 473 print("\nRun ssl_gtests to get a coverage report...") 474 self.runSslGtests(outdir) 475 print("Done.") 476 477 sancov_file = self.findSanCovFile(outdir) 478 if not sancov_file: 479 print("Couldn't find .sancov file.") 480 sys.exit(1) 481 482 symcov_file = os.path.join(outdir, "ssl_gtest.symcov") 483 out = open(symcov_file, 'wb') 484 # Don't exit immediately on error 485 symbol_retcode = subprocess.call([ 486 "sancov", 487 "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"), 488 "-symbolize", sancov_file, 489 os.path.join(cwd, "../dist/Debug/bin/ssl_gtest") 490 ], stdout=out) 491 out.close() 492 493 print("\nCopying ssl_gtests to artifacts...") 494 shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"), 495 os.path.join(outdir, "ssl_gtest")) 496 497 print("\nCoverage report: " + symcov_file) 498 if symbol_retcode > 0: 499 print("sancov failed to symbolize with return code {}".format(symbol_retcode)) 500 sys.exit(symbol_retcode) 501 502class commandsAction(argparse.Action): 503 commands = [] 504 505 def __call__(self, parser, args, values, option_string=None): 506 for c in commandsAction.commands: 507 print(c) 508 509def parse_arguments(): 510 parser = argparse.ArgumentParser( 511 description='NSS helper script. ' + 512 'Make sure to separate sub-command arguments with --.') 513 subparsers = parser.add_subparsers() 514 515 parser_build = subparsers.add_parser( 516 'build', help='All arguments are passed to build.sh') 517 parser_build.add_argument( 518 'build_args', nargs='*', help="build arguments", action=buildAction) 519 520 parser_cf = subparsers.add_parser( 521 'clang-format', 522 help=""" 523 Run clang-format. 524 525 By default this runs against any files that you have modified. If 526 there are no modified files, it checks everything. 527 """) 528 parser_cf.add_argument( 529 '--noroot', 530 help='On linux, suppress the use of \'sudo\' for running docker.', 531 action='store_true') 532 parser_cf.add_argument( 533 '<file/dir>', 534 nargs='*', 535 help="Specify files or directories to run clang-format on", 536 action=cfAction) 537 538 parser_sa = subparsers.add_parser( 539 'static-analysis', 540 help=""" 541 Run static-analysis tools based on coverity. 542 543 By default this runs only on automation and provides a list of issues that 544 are only present locally. 545 """) 546 parser_sa.add_argument( 547 '--config', help='Path to Coverity config file. Only used for local runs.', 548 default=None) 549 parser_sa.add_argument( 550 '--storage', help=""" 551 Path where to store Coverity binaries and results. If none, the base repository will be used. 552 """, 553 default=None) 554 parser_sa.add_argument( 555 '--force', help='Force the re-download of the coverity artefact.', 556 action='store_true') 557 parser_sa.add_argument( 558 '<file>', 559 nargs='*', 560 help="Specify files to run Coverity on. If no files are specified the analysis will check the entire project.", 561 action=coverityAction) 562 563 parser_test = subparsers.add_parser( 564 'tests', help='Run tests through tests/all.sh.') 565 tests = [ 566 "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips", 567 "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec", 568 "gtests", "ssl_gtests", "bogo", "interop", "policy" 569 ] 570 parser_test.add_argument( 571 'test', choices=tests, help="Available tests", action=testAction) 572 573 parser_cov = subparsers.add_parser( 574 'coverage', help='Generate coverage report') 575 cov_modules = ["ssl_gtests"] 576 parser_cov.add_argument( 577 '--outdir', help='Output directory for coverage report data.', 578 default=tempfile.mkdtemp()) 579 parser_cov.add_argument( 580 'module', choices=cov_modules, help="Available coverage modules", 581 action=covAction) 582 583 parser_commands = subparsers.add_parser( 584 'mach-completion', 585 help="list commands") 586 parser_commands.add_argument( 587 'mach-completion', 588 nargs='*', 589 action=commandsAction) 590 591 commandsAction.commands = [c for c in subparsers.choices] 592 return parser.parse_args() 593 594 595def main(): 596 parse_arguments() 597 598 599if __name__ == '__main__': 600 main() 601