1#!/usr/bin/env python3 2# Copyright (c) 2018-2019 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 6import argparse 7import os 8import subprocess 9import sys 10 11def setup(): 12 global args, workdir 13 programs = ['ruby', 'git', 'make', 'wget', 'curl'] 14 if args.kvm: 15 programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] 16 elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'): 17 dockers = ['docker.io', 'docker-ce'] 18 for i in dockers: 19 return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) 20 if return_code == 0: 21 break 22 if return_code != 0: 23 print('Cannot find any way to install Docker.', file=sys.stderr) 24 sys.exit(1) 25 else: 26 programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] 27 subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) 28 if not os.path.isdir('gitian.sigs'): 29 subprocess.check_call(['git', 'clone', 'https://github.com/namecoin/gitian.sigs.git']) 30 if not os.path.isdir('namecoin-detached-sigs'): 31 #subprocess.check_call(['git', 'clone', 'https://github.com/namecoin/namecoin-detached-sigs.git']) 32 print("Cannot clone namecoin-detached-sigs because Namecoin doesn't support detached sigs yet. Will clone other repos.") 33 if not os.path.isdir('gitian-builder'): 34 subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) 35 if not os.path.isdir('namecoin-core'): 36 subprocess.check_call(['git', 'clone', 'https://github.com/namecoin/namecoin-core.git']) 37 os.chdir('gitian-builder') 38 make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] 39 if args.docker: 40 make_image_prog += ['--docker'] 41 elif not args.kvm: 42 make_image_prog += ['--lxc'] 43 subprocess.check_call(make_image_prog) 44 os.chdir(workdir) 45 if args.is_bionic and not args.kvm and not args.docker: 46 subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) 47 print('Reboot is required') 48 sys.exit(0) 49 50def build(): 51 global args, workdir 52 53 os.makedirs('namecoin-binaries/' + args.version, exist_ok=True) 54 print('\nBuilding Dependencies\n') 55 os.chdir('gitian-builder') 56 os.makedirs('inputs', exist_ok=True) 57 58 subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) 59 subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) 60 subprocess.check_call(['make', '-C', '../namecoin-core/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) 61 62 if args.linux: 63 print('\nCompiling ' + args.version + ' Linux') 64 subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'namecoin='+args.commit, '--url', 'namecoin='+args.url, '../namecoin-core/contrib/gitian-descriptors/gitian-linux.yml']) 65 subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-linux.yml']) 66 subprocess.check_call('mv build/out/namecoin-*.tar.gz build/out/src/namecoin-*.tar.gz ../namecoin-binaries/'+args.version, shell=True) 67 68 if args.windows: 69 print('\nCompiling ' + args.version + ' Windows') 70 subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'namecoin='+args.commit, '--url', 'namecoin='+args.url, '../namecoin-core/contrib/gitian-descriptors/gitian-win.yml']) 71 subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-win.yml']) 72 subprocess.check_call('mv build/out/namecoin-*-win-unsigned.tar.gz inputs/', shell=True) 73 subprocess.check_call('mv build/out/namecoin-*.zip build/out/namecoin-*.exe build/out/src/namecoin-*.tar.gz ../namecoin-binaries/'+args.version, shell=True) 74 75 if args.macos: 76 print('\nCompiling ' + args.version + ' MacOS') 77 subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'namecoin='+args.commit, '--url', 'namecoin='+args.url, '../namecoin-core/contrib/gitian-descriptors/gitian-osx.yml']) 78 subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-osx.yml']) 79 subprocess.check_call('mv build/out/namecoin-*-osx-unsigned.tar.gz inputs/', shell=True) 80 subprocess.check_call('mv build/out/namecoin-*.tar.gz build/out/namecoin-*.dmg build/out/src/namecoin-*.tar.gz ../namecoin-binaries/'+args.version, shell=True) 81 82 os.chdir(workdir) 83 84 if args.commit_files: 85 print('\nCommitting '+args.version+' Unsigned Sigs\n') 86 os.chdir('gitian.sigs') 87 subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) 88 subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) 89 subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) 90 subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) 91 os.chdir(workdir) 92 93def sign(): 94 global args, workdir 95 os.chdir('gitian-builder') 96 97 if args.windows: 98 print('\nSigning ' + args.version + ' Windows') 99 subprocess.check_call('cp inputs/namecoin-' + args.version + '-win-unsigned.tar.gz inputs/namecoin-win-unsigned.tar.gz', shell=True) 100 subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../namecoin-core/contrib/gitian-descriptors/gitian-win-signer.yml']) 101 subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-win-signer.yml']) 102 subprocess.check_call('mv build/out/namecoin-*win64-setup.exe ../namecoin-binaries/'+args.version, shell=True) 103 104 if args.macos: 105 print('\nSigning ' + args.version + ' MacOS') 106 subprocess.check_call('cp inputs/namecoin-' + args.version + '-osx-unsigned.tar.gz inputs/namecoin-osx-unsigned.tar.gz', shell=True) 107 subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../namecoin-core/contrib/gitian-descriptors/gitian-osx-signer.yml']) 108 subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-osx-signer.yml']) 109 subprocess.check_call('mv build/out/namecoin-osx-signed.dmg ../namecoin-binaries/'+args.version+'/namecoin-'+args.version+'-osx.dmg', shell=True) 110 111 os.chdir(workdir) 112 113 if args.commit_files: 114 print('\nCommitting '+args.version+' Signed Sigs\n') 115 os.chdir('gitian.sigs') 116 subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) 117 subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) 118 subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) 119 os.chdir(workdir) 120 121def verify(): 122 global args, workdir 123 rc = 0 124 os.chdir('gitian-builder') 125 126 print('\nVerifying v'+args.version+' Linux\n') 127 if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../namecoin-core/contrib/gitian-descriptors/gitian-linux.yml']): 128 print('Verifying v'+args.version+' Linux FAILED\n') 129 rc = 1 130 131 print('\nVerifying v'+args.version+' Windows\n') 132 if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../namecoin-core/contrib/gitian-descriptors/gitian-win.yml']): 133 print('Verifying v'+args.version+' Windows FAILED\n') 134 rc = 1 135 136 print('\nVerifying v'+args.version+' MacOS\n') 137 if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../namecoin-core/contrib/gitian-descriptors/gitian-osx.yml']): 138 print('Verifying v'+args.version+' MacOS FAILED\n') 139 rc = 1 140 141 print('\nVerifying v'+args.version+' Signed Windows\n') 142 if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../namecoin-core/contrib/gitian-descriptors/gitian-win-signer.yml']): 143 print('Verifying v'+args.version+' Signed Windows FAILED\n') 144 rc = 1 145 146 print('\nVerifying v'+args.version+' Signed MacOS\n') 147 if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../namecoin-core/contrib/gitian-descriptors/gitian-osx-signer.yml']): 148 print('Verifying v'+args.version+' Signed MacOS FAILED\n') 149 rc = 1 150 151 os.chdir(workdir) 152 return rc 153 154def main(): 155 global args, workdir 156 157 parser = argparse.ArgumentParser(description='Script for running full Gitian builds.') 158 parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') 159 parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') 160 parser.add_argument('-u', '--url', dest='url', default='https://github.com/namecoin/namecoin-core', help='Specify the URL of the repository. Default is %(default)s') 161 parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') 162 parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') 163 parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') 164 parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') 165 parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') 166 parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') 167 parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') 168 parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') 169 parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') 170 parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)') 171 parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') 172 parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') 173 parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') 174 parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') 175 176 args = parser.parse_args() 177 workdir = os.getcwd() 178 179 args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) 180 181 if args.kvm and args.docker: 182 raise Exception('Error: cannot have both kvm and docker') 183 184 # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they 185 # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm). 186 os.environ['USE_LXC'] = '' 187 os.environ['USE_VBOX'] = '' 188 os.environ['USE_DOCKER'] = '' 189 if args.docker: 190 os.environ['USE_DOCKER'] = '1' 191 elif not args.kvm: 192 os.environ['USE_LXC'] = '1' 193 if 'GITIAN_HOST_IP' not in os.environ.keys(): 194 os.environ['GITIAN_HOST_IP'] = '10.0.3.1' 195 if 'LXC_GUEST_IP' not in os.environ.keys(): 196 os.environ['LXC_GUEST_IP'] = '10.0.3.5' 197 198 if args.setup: 199 setup() 200 201 if args.buildsign: 202 args.build = True 203 args.sign = True 204 205 if not args.build and not args.sign and not args.verify: 206 sys.exit(0) 207 208 args.linux = 'l' in args.os 209 args.windows = 'w' in args.os 210 args.macos = 'm' in args.os 211 212 # Disable for MacOS if no SDK found 213 if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'): 214 print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') 215 args.macos = False 216 217 args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' 218 219 script_name = os.path.basename(sys.argv[0]) 220 if not args.signer: 221 print(script_name+': Missing signer') 222 print('Try '+script_name+' --help for more information') 223 sys.exit(1) 224 if not args.version: 225 print(script_name+': Missing version') 226 print('Try '+script_name+' --help for more information') 227 sys.exit(1) 228 229 # Add leading 'nc' for tags 230 if args.commit and args.pull: 231 raise Exception('Cannot have both commit and pull') 232 args.commit = ('' if args.commit else 'nc') + args.version 233 234 os.chdir('namecoin-core') 235 if args.pull: 236 subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) 237 os.chdir('../gitian-builder/inputs/namecoin') 238 subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) 239 args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() 240 args.version = 'pull-' + args.version 241 print(args.commit) 242 subprocess.check_call(['git', 'fetch']) 243 subprocess.check_call(['git', 'checkout', args.commit]) 244 os.chdir(workdir) 245 246 os.chdir('gitian-builder') 247 subprocess.check_call(['git', 'pull']) 248 os.chdir(workdir) 249 250 if args.build: 251 build() 252 253 if args.sign: 254 sign() 255 256 if args.verify: 257 os.chdir('gitian.sigs') 258 subprocess.check_call(['git', 'pull']) 259 os.chdir(workdir) 260 sys.exit(verify()) 261 262if __name__ == '__main__': 263 main() 264