1#!/usr/bin/env python
2#
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6##########################################################################
7#
8# This is a collection of helper tools to get stuff done in NSS.
9#
10
11import sys
12import argparse
13import fnmatch
14import io
15import subprocess
16import os
17import platform
18import shutil
19import tarfile
20import tempfile
21
22from hashlib import sha256
23
24DEVNULL = open(os.devnull, 'wb')
25cwd = os.path.dirname(os.path.abspath(__file__))
26
27def run_tests(test, cycles="standard", env={}, silent=False):
28    domsuf = os.getenv('DOMSUF', "localdomain")
29    host = os.getenv('HOST', "localhost")
30    env = env.copy()
31    env.update({
32        "NSS_TESTS": test,
33        "NSS_CYCLES": cycles,
34        "DOMSUF": domsuf,
35        "HOST": host
36    })
37    os_env = os.environ
38    os_env.update(env)
39    command = cwd + "/tests/all.sh"
40    stdout = stderr = DEVNULL if silent else None
41    subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr)
42
43
44class coverityAction(argparse.Action):
45
46    def get_coverity_remote_cfg(self):
47        secret_name = 'project/relman/coverity-nss'
48        secrets_url = 'http://taskcluster/secrets/v1/secret/{}'.format(secret_name)
49
50        print('Using symbol upload token from the secrets service: "{}"'.
51              format(secrets_url))
52
53        import requests
54        res = requests.get(secrets_url)
55        res.raise_for_status()
56        secret = res.json()
57        cov_config = secret['secret'] if 'secret' in secret else None
58
59        if cov_config is None:
60            print('Ill formatted secret for Coverity. Aborting analysis.')
61            return None
62
63        return cov_config
64
65    def get_coverity_local_cfg(self, path):
66        try:
67            import yaml
68            file_handler = open(path)
69            config = yaml.safe_load(file_handler)
70        except Exception:
71            print('Unable to load coverity config from {}'.format(path))
72            return None
73        return config
74
75    def get_cov_config(self, path):
76        cov_config = None
77        if self.local_config:
78            cov_config = self.get_coverity_local_cfg(path)
79        else:
80            cov_config = self.get_coverity_remote_cfg()
81
82        if cov_config is None:
83            print('Unable to load Coverity config.')
84            return 1
85
86        self.cov_analysis_url = cov_config.get('package_url')
87        self.cov_package_name = cov_config.get('package_name')
88        self.cov_url = cov_config.get('server_url')
89        self.cov_port = cov_config.get('server_port')
90        self.cov_auth = cov_config.get('auth_key')
91        self.cov_package_ver = cov_config.get('package_ver')
92        self.cov_full_stack = cov_config.get('full_stack', False)
93
94        return 0
95
96    def download_coverity(self):
97        if self.cov_url is None or self.cov_port is None or self.cov_analysis_url is None or self.cov_auth is None:
98            print('Missing Coverity config options!')
99            return 1
100
101        COVERITY_CONFIG = '''
102        {
103            "type": "Coverity configuration",
104            "format_version": 1,
105            "settings": {
106            "server": {
107                "host": "%s",
108                "port": %s,
109                "ssl" : true,
110                "on_new_cert" : "trust",
111                "auth_key_file": "%s"
112            },
113            "stream": "NSS",
114            "cov_run_desktop": {
115                "build_cmd": ["%s"],
116                "clean_cmd": ["%s", "-cc"],
117            }
118            }
119        }
120        '''
121        # Generate the coverity.conf and auth files
122        build_cmd = os.path.join(cwd, 'build.sh')
123        cov_auth_path = os.path.join(self.cov_state_path, 'auth')
124        cov_setup_path = os.path.join(self.cov_state_path, 'coverity.conf')
125        cov_conf = COVERITY_CONFIG % (self.cov_url, self.cov_port, cov_auth_path, build_cmd, build_cmd)
126
127        def download(artifact_url, target):
128            import requests
129            resp = requests.get(artifact_url, verify=False, stream=True)
130            resp.raise_for_status()
131
132            # Extract archive into destination
133            with tarfile.open(fileobj=io.BytesIO(resp.content)) as tar:
134                tar.extractall(target)
135
136        download(self.cov_analysis_url, self.cov_state_path)
137
138        with open(cov_auth_path, 'w') as f:
139            f.write(self.cov_auth)
140
141        # Modify it's permission to 600
142        os.chmod(cov_auth_path, 0o600)
143
144        with open(cov_setup_path, 'a') as f:
145            f.write(cov_conf)
146
147    def setup_coverity(self, config_path, storage_path=None, force_download=True):
148        rc = self.get_cov_config(config_path)
149
150        if rc != 0:
151            return rc
152
153        if storage_path is None:
154            # If storage_path is None we set the context of the coverity into the cwd.
155            storage_path = cwd
156
157        self.cov_state_path = os.path.join(storage_path, "coverity")
158
159        if force_download is True or not os.path.exists(self.cov_state_path):
160            shutil.rmtree(self.cov_state_path, ignore_errors=True)
161            os.mkdir(self.cov_state_path)
162
163            # Download everything that we need for Coverity from out private instance
164            self.download_coverity()
165
166        self.cov_path = os.path.join(self.cov_state_path, self.cov_package_name)
167        self.cov_run_desktop = os.path.join(self.cov_path, 'bin', 'cov-run-desktop')
168        self.cov_translate = os.path.join(self.cov_path, 'bin', 'cov-translate')
169        self.cov_configure = os.path.join(self.cov_path, 'bin', 'cov-configure')
170        self.cov_work_path = os.path.join(self.cov_state_path, 'data-coverity')
171        self.cov_idir_path = os.path.join(self.cov_work_path, self.cov_package_ver, 'idir')
172
173        if not os.path.exists(self.cov_path) or \
174           not os.path.exists(self.cov_run_desktop) or \
175           not os.path.exists(self.cov_translate) or \
176           not os.path.exists(self.cov_configure):
177            print('Missing Coverity in {}'.format(self.cov_path))
178            return 1
179
180        return 0
181
182    def run_process(self, args, cwd=cwd):
183        proc = subprocess.Popen(args, cwd=cwd)
184        status = None
185        while status is None:
186            try:
187                status = proc.wait()
188            except KeyboardInterrupt:
189                pass
190        return status
191
192    def cov_is_file_in_source(self, abs_path):
193        if os.path.islink(abs_path):
194            abs_path = os.path.realpath(abs_path)
195        return abs_path
196
197    def dump_cov_artifact(self, cov_results, source, output):
198        import json
199
200        def relpath(path):
201            '''Build path relative to repository root'''
202            if path.startswith(cwd):
203                return os.path.relpath(path, cwd)
204            return path
205
206        # Parse Coverity json into structured issues
207        with open(cov_results) as f:
208            result = json.load(f)
209
210            # Parse the issues to a standard json format
211            issues_dict = {'files': {}}
212
213            files_list = issues_dict['files']
214
215            def build_element(issue):
216                # We look only for main event
217                event_path = next((event for event in issue['events'] if event['main'] is True), None)
218
219                dict_issue = {
220                    'line': issue['mainEventLineNumber'],
221                    'flag': issue['checkerName'],
222                    'message': event_path['eventDescription'],
223                    'extra': {
224                        'category': issue['checkerProperties']['category'],
225                        'stateOnServer': issue['stateOnServer'],
226                        'stack': []
227                    }
228                }
229
230                # Embed all events into extra message
231                for event in issue['events']:
232                    dict_issue['extra']['stack'].append({'file_path': relpath(event['strippedFilePathname']),
233                                                         'line_number': event['lineNumber'],
234                                                         'path_type': event['eventTag'],
235                                                         'description': event['eventDescription']})
236
237                return dict_issue
238
239            for issue in result['issues']:
240                path = self.cov_is_file_in_source(issue['strippedMainEventFilePathname'])
241                if path is None:
242                    # Since we skip a result we should log it
243                    print('Skipping CID: {0} from file: {1} since it\'s not related with the current patch.'.format(
244                        issue['stateOnServer']['cid'], issue['strippedMainEventFilePathname']))
245                    continue
246                # If path does not start with `cwd` skip it
247                if not path.startswith(cwd):
248                    continue
249                path = relpath(path)
250                if path in files_list:
251                    files_list[path]['warnings'].append(build_element(issue))
252                else:
253                    files_list[path] = {'warnings': [build_element(issue)]}
254
255            with open(output, 'w') as f:
256                json.dump(issues_dict, f)
257
258    def mutate_paths(self, paths):
259        for index in xrange(len(paths)):
260            paths[index] = os.path.abspath(paths[index])
261
262    def __call__(self, parser, args, paths, option_string=None):
263        self.local_config = True
264        config_path = args.config
265        storage_path = args.storage
266
267        have_paths = True
268        if len(paths) == 0:
269            have_paths = False
270            print('No files have been specified for analysis, running Coverity on the entire project.')
271
272        self.mutate_paths(paths)
273
274        if config_path is None:
275            self.local_config = False
276            print('No coverity config path has been specified, so running in automation.')
277            if 'NSS_AUTOMATION' not in os.environ:
278                print('Coverity based static-analysis cannot be ran outside automation.')
279                return 1
280
281        rc = self.setup_coverity(config_path, storage_path, args.force)
282        if rc != 0:
283            return 1
284
285        # First run cov-run-desktop --setup in order to setup the analysis env
286        cmd = [self.cov_run_desktop, '--setup']
287        print('Running {} --setup'.format(self.cov_run_desktop))
288
289        rc = self.run_process(args=cmd, cwd=self.cov_path)
290
291        if rc != 0:
292            print('Running {} --setup failed!'.format(self.cov_run_desktop))
293            return rc
294
295        cov_result = os.path.join(self.cov_state_path, 'cov-results.json')
296
297        # Once the capture is performed we need to do the actual Coverity Desktop analysis
298        if have_paths:
299            cmd = [self.cov_run_desktop, '--json-output-v6', cov_result] + paths
300        else:
301            cmd = [self.cov_run_desktop, '--json-output-v6', cov_result, '--analyze-captured-source']
302
303        print('Running Coverity Analysis for {}'.format(cmd))
304
305        rc = self.run_process(cmd, cwd=self.cov_state_path)
306
307        if rc != 0:
308            print('Coverity Analysis failed!')
309
310        # On automation, like try, we want to build an artifact with the results.
311        if 'NSS_AUTOMATION' in os.environ:
312            self.dump_cov_artifact(cov_result, cov_result, "/home/worker/nss/coverity/coverity.json")
313
314
315class cfAction(argparse.Action):
316    docker_command = None
317    restorecon = None
318
319    def __call__(self, parser, args, values, option_string=None):
320        self.setDockerCommand(args)
321
322        if values:
323            files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values]
324        else:
325            files = self.modifiedFiles()
326
327        # First check if we can run docker.
328        try:
329            with open(os.devnull, "w") as f:
330                subprocess.check_call(
331                    self.docker_command + ["images"], stdout=f)
332        except:
333            self.docker_command = None
334
335        if self.docker_command is None:
336            print("warning: running clang-format directly, which isn't guaranteed to be correct")
337            command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files
338            repr(command)
339            subprocess.call(command)
340            return
341
342        files = [os.path.join('/home/worker/nss', x) for x in files]
343        docker_image = 'clang-format-service:latest'
344        cf_docker_folder = cwd + "/automation/clang-format"
345
346        # Build the image if necessary.
347        if self.filesChanged(cf_docker_folder):
348            self.buildImage(docker_image, cf_docker_folder)
349
350        # Check if we have the docker image.
351        try:
352            command = self.docker_command + [
353                "image", "inspect", "clang-format-service:latest"
354            ]
355            with open(os.devnull, "w") as f:
356                subprocess.check_call(command, stdout=f)
357        except:
358            print("I have to build the docker image first.")
359            self.buildImage(docker_image, cf_docker_folder)
360
361        command = self.docker_command + [
362            'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image
363        ]
364        # The clang format script returns 1 if something's to do. We don't
365        # care.
366        subprocess.call(command + files)
367        if self.restorecon is not None:
368            subprocess.call([self.restorecon, '-R', cwd])
369
370    def filesChanged(self, path):
371        hash = sha256()
372        for dirname, dirnames, files in os.walk(path):
373            for file in files:
374                with open(os.path.join(dirname, file), "rb") as f:
375                    hash.update(f.read())
376        chk_file = cwd + "/.chk"
377        old_chk = ""
378        new_chk = hash.hexdigest()
379        if os.path.exists(chk_file):
380            with open(chk_file) as f:
381                old_chk = f.readline()
382        if old_chk != new_chk:
383            with open(chk_file, "w+") as f:
384                f.write(new_chk)
385            return True
386        return False
387
388    def buildImage(self, docker_image, cf_docker_folder):
389        command = self.docker_command + [
390            "build", "-t", docker_image, cf_docker_folder
391        ]
392        subprocess.check_call(command)
393        return
394
395    def setDockerCommand(self, args):
396        from distutils.spawn import find_executable
397        if platform.system() == "Linux":
398            self.restorecon = find_executable("restorecon")
399        dcmd = find_executable("docker")
400        if dcmd is not None:
401            self.docker_command = [dcmd]
402            if not args.noroot:
403                self.docker_command = ["sudo"] + self.docker_command
404        else:
405            self.docker_command = None
406
407    def modifiedFiles(self):
408        files = []
409        if os.path.exists(os.path.join(cwd, '.hg')):
410            st = subprocess.Popen(['hg', 'status', '-m', '-a'],
411                                  cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
412            for line in iter(st.stdout.readline, ''):
413                files += [line[2:].rstrip()]
414        elif os.path.exists(os.path.join(cwd, '.git')):
415            st = subprocess.Popen(['git', 'status', '--porcelain'],
416                                  cwd=cwd, stdout=subprocess.PIPE)
417            for line in iter(st.stdout.readline, ''):
418                if line[1] == 'M' or line[1] != 'D' and \
419                        (line[0] == 'M' or line[0] == 'A' or
420                         line[0] == 'C' or line[0] == 'U'):
421                    files += [line[3:].rstrip()]
422                elif line[0] == 'R':
423                    files += [line[line.index(' -> ', beg=4) + 4:]]
424        else:
425            print('Warning: neither mercurial nor git detected!')
426
427        def isFormatted(x):
428            return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h'
429        return [x for x in files if isFormatted(x)]
430
431
432class buildAction(argparse.Action):
433
434    def __call__(self, parser, args, values, option_string=None):
435        subprocess.check_call([cwd + "/build.sh"] + values)
436
437
438class testAction(argparse.Action):
439
440    def __call__(self, parser, args, values, option_string=None):
441        run_tests(values)
442
443
444class covAction(argparse.Action):
445
446    def runSslGtests(self, outdir):
447        env = {
448            "GTESTFILTER": "*", # Prevent parallel test runs.
449            "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir,
450            "NSS_DEFAULT_DB_TYPE": "dbm"
451        }
452
453        run_tests("ssl_gtests", env=env, silent=True)
454
455    def findSanCovFile(self, outdir):
456        for file in os.listdir(outdir):
457            if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'):
458                return os.path.join(outdir, file)
459
460        return None
461
462    def __call__(self, parser, args, values, option_string=None):
463        outdir = args.outdir
464        print("Output directory: " + outdir)
465
466        print("\nBuild with coverage sanitizers...\n")
467        sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
468        subprocess.check_call([
469            os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
470            "--sancov=" + sancov_args
471        ])
472
473        print("\nRun ssl_gtests to get a coverage report...")
474        self.runSslGtests(outdir)
475        print("Done.")
476
477        sancov_file = self.findSanCovFile(outdir)
478        if not sancov_file:
479            print("Couldn't find .sancov file.")
480            sys.exit(1)
481
482        symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
483        out = open(symcov_file, 'wb')
484        # Don't exit immediately on error
485        symbol_retcode = subprocess.call([
486            "sancov",
487            "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
488            "-symbolize", sancov_file,
489            os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
490        ], stdout=out)
491        out.close()
492
493        print("\nCopying ssl_gtests to artifacts...")
494        shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"),
495                        os.path.join(outdir, "ssl_gtest"))
496
497        print("\nCoverage report: " + symcov_file)
498        if symbol_retcode > 0:
499            print("sancov failed to symbolize with return code {}".format(symbol_retcode))
500        sys.exit(symbol_retcode)
501
502class commandsAction(argparse.Action):
503    commands = []
504
505    def __call__(self, parser, args, values, option_string=None):
506        for c in commandsAction.commands:
507            print(c)
508
509def parse_arguments():
510    parser = argparse.ArgumentParser(
511        description='NSS helper script. ' +
512        'Make sure to separate sub-command arguments with --.')
513    subparsers = parser.add_subparsers()
514
515    parser_build = subparsers.add_parser(
516        'build', help='All arguments are passed to build.sh')
517    parser_build.add_argument(
518        'build_args', nargs='*', help="build arguments", action=buildAction)
519
520    parser_cf = subparsers.add_parser(
521        'clang-format',
522        help="""
523        Run clang-format.
524
525        By default this runs against any files that you have modified.  If
526        there are no modified files, it checks everything.
527        """)
528    parser_cf.add_argument(
529        '--noroot',
530        help='On linux, suppress the use of \'sudo\' for running docker.',
531        action='store_true')
532    parser_cf.add_argument(
533        '<file/dir>',
534        nargs='*',
535        help="Specify files or directories to run clang-format on",
536        action=cfAction)
537
538    parser_sa = subparsers.add_parser(
539        'static-analysis',
540        help="""
541        Run static-analysis tools based on coverity.
542
543        By default this runs only on automation and provides a list of issues that
544        are only present locally.
545        """)
546    parser_sa.add_argument(
547        '--config', help='Path to Coverity config file. Only used for local runs.',
548        default=None)
549    parser_sa.add_argument(
550        '--storage', help="""
551        Path where to store Coverity binaries and results. If none, the base repository will be used.
552        """,
553        default=None)
554    parser_sa.add_argument(
555        '--force', help='Force the re-download of the coverity artefact.',
556        action='store_true')
557    parser_sa.add_argument(
558        '<file>',
559        nargs='*',
560        help="Specify files to run Coverity on. If no files are specified the analysis will check the entire project.",
561        action=coverityAction)
562
563    parser_test = subparsers.add_parser(
564        'tests', help='Run tests through tests/all.sh.')
565    tests = [
566        "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
567        "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
568        "gtests", "ssl_gtests", "bogo", "interop", "policy"
569    ]
570    parser_test.add_argument(
571        'test', choices=tests, help="Available tests", action=testAction)
572
573    parser_cov = subparsers.add_parser(
574        'coverage', help='Generate coverage report')
575    cov_modules = ["ssl_gtests"]
576    parser_cov.add_argument(
577        '--outdir', help='Output directory for coverage report data.',
578        default=tempfile.mkdtemp())
579    parser_cov.add_argument(
580        'module', choices=cov_modules, help="Available coverage modules",
581        action=covAction)
582
583    parser_commands = subparsers.add_parser(
584        'mach-completion',
585        help="list commands")
586    parser_commands.add_argument(
587        'mach-completion',
588        nargs='*',
589        action=commandsAction)
590
591    commandsAction.commands = [c for c in subparsers.choices]
592    return parser.parse_args()
593
594
595def main():
596    parse_arguments()
597
598
599if __name__ == '__main__':
600    main()
601