1#!/usr/bin/env python3
2# Copyright (c) 2018-2019 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Verify commits against a trusted keys list."""
6import argparse
7import hashlib
8import logging
9import os
10import subprocess
11import sys
12import time
13
14GIT = os.getenv('GIT', 'git')
15
16def tree_sha512sum(commit='HEAD'):
17    """Calculate the Tree-sha512 for the commit.
18
19    This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools."""
20
21    # request metadata for entire tree, recursively
22    files = []
23    blob_by_name = {}
24    for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
25        name_sep = line.index(b'\t')
26        metadata = line[:name_sep].split()  # perms, 'blob', blobid
27        assert metadata[1] == b'blob'
28        name = line[name_sep + 1:]
29        files.append(name)
30        blob_by_name[name] = metadata[2]
31
32    files.sort()
33    # open connection to git-cat-file in batch mode to request data for all blobs
34    # this is much faster than launching it per file
35    p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
36    overall = hashlib.sha512()
37    for f in files:
38        blob = blob_by_name[f]
39        # request blob
40        p.stdin.write(blob + b'\n')
41        p.stdin.flush()
42        # read header: blob, "blob", size
43        reply = p.stdout.readline().split()
44        assert reply[0] == blob and reply[1] == b'blob'
45        size = int(reply[2])
46        # hash the blob data
47        intern = hashlib.sha512()
48        ptr = 0
49        while ptr < size:
50            bs = min(65536, size - ptr)
51            piece = p.stdout.read(bs)
52            if len(piece) == bs:
53                intern.update(piece)
54            else:
55                raise IOError('Premature EOF reading git cat-file output')
56            ptr += bs
57        dig = intern.hexdigest()
58        assert p.stdout.read(1) == b'\n'  # ignore LF that follows blob data
59        # update overall hash with file hash
60        overall.update(dig.encode("utf-8"))
61        overall.update("  ".encode("utf-8"))
62        overall.update(f)
63        overall.update("\n".encode("utf-8"))
64    p.stdin.close()
65    if p.wait():
66        raise IOError('Non-zero return value executing git cat-file')
67    return overall.hexdigest()
68
69def main():
70
71    # Enable debug logging if running in CI
72    if 'CI' in os.environ and os.environ['CI'].lower() == "true":
73        logging.getLogger().setLevel(logging.DEBUG)
74
75    # Parse arguments
76    parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]')
77    parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check')
78    parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER')
79    parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>')
80    args = parser.parse_args()
81
82    # get directory of this program and read data files
83    dirname = os.path.dirname(os.path.abspath(__file__))
84    print("Using verify-commits data from " + dirname)
85    verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0]
86    verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0]
87    revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines()
88    unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines()
89    incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines()
90
91    # Set commit and branch and set variables
92    current_commit = args.commit
93    if ' ' in current_commit:
94        print("Commit must not contain spaces", file=sys.stderr)
95        sys.exit(1)
96    verify_tree = args.verify_tree
97    no_sha1 = True
98    prev_commit = ""
99    initial_commit = current_commit
100    branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit]).decode('utf8').splitlines()[0]
101
102    # Iterate through commits
103    while True:
104
105        # Log a message to prevent Travis from timing out
106        logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8]))
107
108        if current_commit == verified_root:
109            print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root))
110            sys.exit(0)
111        if current_commit == verified_sha512_root:
112            if verify_tree:
113                print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr)
114            verify_tree = False
115            no_sha1 = False
116
117        os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1"
118        os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0"
119
120        # Check that the commit (and parents) was signed with a trusted key
121        if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL):
122            if prev_commit != "":
123                print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
124                print("Parents are:", file=sys.stderr)
125                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ')
126                for parent in parents:
127                    subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
128            else:
129                print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr)
130            sys.exit(1)
131
132        # Check the Tree-SHA512
133        if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
134            tree_hash = tree_sha512sum(current_commit)
135            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines():
136                print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
137                sys.exit(1)
138
139        # Merge commits should only have two parents
140        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ')
141        if len(parents) > 2:
142            print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
143            sys.exit(1)
144
145        # Check that the merge commit is clean
146        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0])
147        check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60  # Only check commits in clean_merge days
148        allow_unclean = current_commit in unclean_merge_allowed
149        if len(parents) == 2 and check_merge and not allow_unclean:
150            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0]
151            subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]])
152            subprocess.call([GIT, 'merge', '--no-ff', '--quiet', '--no-gpg-sign', parents[1]], stdout=subprocess.DEVNULL)
153            recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD']).decode('utf8').splitlines()[0]
154            if current_tree != recreated_tree:
155                print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
156                subprocess.call([GIT, 'diff', current_commit])
157                subprocess.call([GIT, 'checkout', '--force', '--quiet', branch])
158                sys.exit(1)
159            subprocess.call([GIT, 'checkout', '--force', '--quiet', branch])
160
161        prev_commit = current_commit
162        current_commit = parents[0]
163
164if __name__ == '__main__':
165    main()
166