1#!/usr/bin/env python3 2# Copyright (c) 2018-2019 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5"""Verify commits against a trusted keys list.""" 6import argparse 7import hashlib 8import logging 9import os 10import subprocess 11import sys 12import time 13 14GIT = os.getenv('GIT', 'git') 15 16def tree_sha512sum(commit='HEAD'): 17 """Calculate the Tree-sha512 for the commit. 18 19 This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools.""" 20 21 # request metadata for entire tree, recursively 22 files = [] 23 blob_by_name = {} 24 for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): 25 name_sep = line.index(b'\t') 26 metadata = line[:name_sep].split() # perms, 'blob', blobid 27 assert metadata[1] == b'blob' 28 name = line[name_sep + 1:] 29 files.append(name) 30 blob_by_name[name] = metadata[2] 31 32 files.sort() 33 # open connection to git-cat-file in batch mode to request data for all blobs 34 # this is much faster than launching it per file 35 p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) 36 overall = hashlib.sha512() 37 for f in files: 38 blob = blob_by_name[f] 39 # request blob 40 p.stdin.write(blob + b'\n') 41 p.stdin.flush() 42 # read header: blob, "blob", size 43 reply = p.stdout.readline().split() 44 assert reply[0] == blob and reply[1] == b'blob' 45 size = int(reply[2]) 46 # hash the blob data 47 intern = hashlib.sha512() 48 ptr = 0 49 while ptr < size: 50 bs = min(65536, size - ptr) 51 piece = p.stdout.read(bs) 52 if len(piece) == bs: 53 intern.update(piece) 54 else: 55 raise IOError('Premature EOF reading git cat-file output') 56 ptr += bs 57 dig = intern.hexdigest() 58 assert p.stdout.read(1) == b'\n' # ignore LF that follows blob data 59 # update overall hash with file hash 60 overall.update(dig.encode("utf-8")) 61 overall.update(" ".encode("utf-8")) 62 overall.update(f) 63 overall.update("\n".encode("utf-8")) 64 p.stdin.close() 65 if p.wait(): 66 raise IOError('Non-zero return value executing git cat-file') 67 return overall.hexdigest() 68 69def main(): 70 71 # Enable debug logging if running in CI 72 if 'CI' in os.environ and os.environ['CI'].lower() == "true": 73 logging.getLogger().setLevel(logging.DEBUG) 74 75 # Parse arguments 76 parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]') 77 parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check') 78 parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER') 79 parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>') 80 args = parser.parse_args() 81 82 # get directory of this program and read data files 83 dirname = os.path.dirname(os.path.abspath(__file__)) 84 print("Using verify-commits data from " + dirname) 85 verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] 86 verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] 87 revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() 88 unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() 89 incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() 90 91 # Set commit and branch and set variables 92 current_commit = args.commit 93 if ' ' in current_commit: 94 print("Commit must not contain spaces", file=sys.stderr) 95 sys.exit(1) 96 verify_tree = args.verify_tree 97 no_sha1 = True 98 prev_commit = "" 99 initial_commit = current_commit 100 branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit]).decode('utf8').splitlines()[0] 101 102 # Iterate through commits 103 while True: 104 105 # Log a message to prevent Travis from timing out 106 logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8])) 107 108 if current_commit == verified_root: 109 print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) 110 sys.exit(0) 111 if current_commit == verified_sha512_root: 112 if verify_tree: 113 print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr) 114 verify_tree = False 115 no_sha1 = False 116 117 os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1" 118 os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0" 119 120 # Check that the commit (and parents) was signed with a trusted key 121 if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL): 122 if prev_commit != "": 123 print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr) 124 print("Parents are:", file=sys.stderr) 125 parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ') 126 for parent in parents: 127 subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr) 128 else: 129 print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr) 130 sys.exit(1) 131 132 # Check the Tree-SHA512 133 if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed: 134 tree_hash = tree_sha512sum(current_commit) 135 if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines(): 136 print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr) 137 sys.exit(1) 138 139 # Merge commits should only have two parents 140 parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ') 141 if len(parents) > 2: 142 print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr) 143 sys.exit(1) 144 145 # Check that the merge commit is clean 146 commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0]) 147 check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days 148 allow_unclean = current_commit in unclean_merge_allowed 149 if len(parents) == 2 and check_merge and not allow_unclean: 150 current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0] 151 subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]]) 152 subprocess.call([GIT, 'merge', '--no-ff', '--quiet', '--no-gpg-sign', parents[1]], stdout=subprocess.DEVNULL) 153 recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD']).decode('utf8').splitlines()[0] 154 if current_tree != recreated_tree: 155 print("Merge commit {} is not clean".format(current_commit), file=sys.stderr) 156 subprocess.call([GIT, 'diff', current_commit]) 157 subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) 158 sys.exit(1) 159 subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) 160 161 prev_commit = current_commit 162 current_commit = parents[0] 163 164if __name__ == '__main__': 165 main() 166