1#!/usr/bin/env python3
2# Copyright (c) 2018-2019 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6import argparse
7import os
8import subprocess
9import sys
10
11def setup():
12    global args, workdir
13    programs = ['ruby', 'git', 'make', 'wget', 'curl']
14    if args.kvm:
15        programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils']
16    elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'):
17        dockers = ['docker.io', 'docker-ce']
18        for i in dockers:
19            return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i])
20            if return_code == 0:
21                break
22        if return_code != 0:
23            print('Cannot find any way to install Docker.', file=sys.stderr)
24            sys.exit(1)
25    else:
26        programs += ['apt-cacher-ng', 'lxc', 'debootstrap']
27    subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs)
28    if not os.path.isdir('gitian.sigs'):
29        subprocess.check_call(['git', 'clone', 'https://github.com/namecoin/gitian.sigs.git'])
30    if not os.path.isdir('namecoin-detached-sigs'):
31        #subprocess.check_call(['git', 'clone', 'https://github.com/namecoin/namecoin-detached-sigs.git'])
32        print("Cannot clone namecoin-detached-sigs because Namecoin doesn't support detached sigs yet.  Will clone other repos.")
33    if not os.path.isdir('gitian-builder'):
34        subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git'])
35    if not os.path.isdir('namecoin-core'):
36        subprocess.check_call(['git', 'clone', 'https://github.com/namecoin/namecoin-core.git'])
37    os.chdir('gitian-builder')
38    make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64']
39    if args.docker:
40        make_image_prog += ['--docker']
41    elif not args.kvm:
42        make_image_prog += ['--lxc']
43    subprocess.check_call(make_image_prog)
44    os.chdir(workdir)
45    if args.is_bionic and not args.kvm and not args.docker:
46        subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net'])
47        print('Reboot is required')
48        sys.exit(0)
49
50def build():
51    global args, workdir
52
53    os.makedirs('namecoin-binaries/' + args.version, exist_ok=True)
54    print('\nBuilding Dependencies\n')
55    os.chdir('gitian-builder')
56    os.makedirs('inputs', exist_ok=True)
57
58    subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz'])
59    subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True)
60    subprocess.check_call(['make', '-C', '../namecoin-core/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common'])
61
62    if args.linux:
63        print('\nCompiling ' + args.version + ' Linux')
64        subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'namecoin='+args.commit, '--url', 'namecoin='+args.url, '../namecoin-core/contrib/gitian-descriptors/gitian-linux.yml'])
65        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-linux.yml'])
66        subprocess.check_call('mv build/out/namecoin-*.tar.gz build/out/src/namecoin-*.tar.gz ../namecoin-binaries/'+args.version, shell=True)
67
68    if args.windows:
69        print('\nCompiling ' + args.version + ' Windows')
70        subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'namecoin='+args.commit, '--url', 'namecoin='+args.url, '../namecoin-core/contrib/gitian-descriptors/gitian-win.yml'])
71        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-win.yml'])
72        subprocess.check_call('mv build/out/namecoin-*-win-unsigned.tar.gz inputs/', shell=True)
73        subprocess.check_call('mv build/out/namecoin-*.zip build/out/namecoin-*.exe build/out/src/namecoin-*.tar.gz ../namecoin-binaries/'+args.version, shell=True)
74
75    if args.macos:
76        print('\nCompiling ' + args.version + ' MacOS')
77        subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'namecoin='+args.commit, '--url', 'namecoin='+args.url, '../namecoin-core/contrib/gitian-descriptors/gitian-osx.yml'])
78        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-osx.yml'])
79        subprocess.check_call('mv build/out/namecoin-*-osx-unsigned.tar.gz inputs/', shell=True)
80        subprocess.check_call('mv build/out/namecoin-*.tar.gz build/out/namecoin-*.dmg build/out/src/namecoin-*.tar.gz ../namecoin-binaries/'+args.version, shell=True)
81
82    os.chdir(workdir)
83
84    if args.commit_files:
85        print('\nCommitting '+args.version+' Unsigned Sigs\n')
86        os.chdir('gitian.sigs')
87        subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer])
88        subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer])
89        subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer])
90        subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer])
91        os.chdir(workdir)
92
93def sign():
94    global args, workdir
95    os.chdir('gitian-builder')
96
97    if args.windows:
98        print('\nSigning ' + args.version + ' Windows')
99        subprocess.check_call('cp inputs/namecoin-' + args.version + '-win-unsigned.tar.gz inputs/namecoin-win-unsigned.tar.gz', shell=True)
100        subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../namecoin-core/contrib/gitian-descriptors/gitian-win-signer.yml'])
101        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-win-signer.yml'])
102        subprocess.check_call('mv build/out/namecoin-*win64-setup.exe ../namecoin-binaries/'+args.version, shell=True)
103
104    if args.macos:
105        print('\nSigning ' + args.version + ' MacOS')
106        subprocess.check_call('cp inputs/namecoin-' + args.version + '-osx-unsigned.tar.gz inputs/namecoin-osx-unsigned.tar.gz', shell=True)
107        subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../namecoin-core/contrib/gitian-descriptors/gitian-osx-signer.yml'])
108        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../namecoin-core/contrib/gitian-descriptors/gitian-osx-signer.yml'])
109        subprocess.check_call('mv build/out/namecoin-osx-signed.dmg ../namecoin-binaries/'+args.version+'/namecoin-'+args.version+'-osx.dmg', shell=True)
110
111    os.chdir(workdir)
112
113    if args.commit_files:
114        print('\nCommitting '+args.version+' Signed Sigs\n')
115        os.chdir('gitian.sigs')
116        subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer])
117        subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer])
118        subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer])
119        os.chdir(workdir)
120
121def verify():
122    global args, workdir
123    rc = 0
124    os.chdir('gitian-builder')
125
126    print('\nVerifying v'+args.version+' Linux\n')
127    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../namecoin-core/contrib/gitian-descriptors/gitian-linux.yml']):
128        print('Verifying v'+args.version+' Linux FAILED\n')
129        rc = 1
130
131    print('\nVerifying v'+args.version+' Windows\n')
132    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../namecoin-core/contrib/gitian-descriptors/gitian-win.yml']):
133        print('Verifying v'+args.version+' Windows FAILED\n')
134        rc = 1
135
136    print('\nVerifying v'+args.version+' MacOS\n')
137    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../namecoin-core/contrib/gitian-descriptors/gitian-osx.yml']):
138        print('Verifying v'+args.version+' MacOS FAILED\n')
139        rc = 1
140
141    print('\nVerifying v'+args.version+' Signed Windows\n')
142    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../namecoin-core/contrib/gitian-descriptors/gitian-win-signer.yml']):
143        print('Verifying v'+args.version+' Signed Windows FAILED\n')
144        rc = 1
145
146    print('\nVerifying v'+args.version+' Signed MacOS\n')
147    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../namecoin-core/contrib/gitian-descriptors/gitian-osx-signer.yml']):
148        print('Verifying v'+args.version+' Signed MacOS FAILED\n')
149        rc = 1
150
151    os.chdir(workdir)
152    return rc
153
154def main():
155    global args, workdir
156
157    parser = argparse.ArgumentParser(description='Script for running full Gitian builds.')
158    parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch')
159    parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request')
160    parser.add_argument('-u', '--url', dest='url', default='https://github.com/namecoin/namecoin-core', help='Specify the URL of the repository. Default is %(default)s')
161    parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build')
162    parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build')
163    parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS')
164    parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries')
165    parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS')
166    parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s')
167    parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s')
168    parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC')
169    parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC')
170    parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)')
171    parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.')
172    parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git')
173    parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file')
174    parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified')
175
176    args = parser.parse_args()
177    workdir = os.getcwd()
178
179    args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs'])
180
181    if args.kvm and args.docker:
182        raise Exception('Error: cannot have both kvm and docker')
183
184    # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they
185    # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm).
186    os.environ['USE_LXC'] = ''
187    os.environ['USE_VBOX'] = ''
188    os.environ['USE_DOCKER'] = ''
189    if args.docker:
190        os.environ['USE_DOCKER'] = '1'
191    elif not args.kvm:
192        os.environ['USE_LXC'] = '1'
193        if 'GITIAN_HOST_IP' not in os.environ.keys():
194            os.environ['GITIAN_HOST_IP'] = '10.0.3.1'
195        if 'LXC_GUEST_IP' not in os.environ.keys():
196            os.environ['LXC_GUEST_IP'] = '10.0.3.5'
197
198    if args.setup:
199        setup()
200
201    if args.buildsign:
202        args.build = True
203        args.sign = True
204
205    if not args.build and not args.sign and not args.verify:
206        sys.exit(0)
207
208    args.linux = 'l' in args.os
209    args.windows = 'w' in args.os
210    args.macos = 'm' in args.os
211
212    # Disable for MacOS if no SDK found
213    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'):
214        print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
215        args.macos = False
216
217    args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
218
219    script_name = os.path.basename(sys.argv[0])
220    if not args.signer:
221        print(script_name+': Missing signer')
222        print('Try '+script_name+' --help for more information')
223        sys.exit(1)
224    if not args.version:
225        print(script_name+': Missing version')
226        print('Try '+script_name+' --help for more information')
227        sys.exit(1)
228
229    # Add leading 'nc' for tags
230    if args.commit and args.pull:
231        raise Exception('Cannot have both commit and pull')
232    args.commit = ('' if args.commit else 'nc') + args.version
233
234    os.chdir('namecoin-core')
235    if args.pull:
236        subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
237        os.chdir('../gitian-builder/inputs/namecoin')
238        subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
239        args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip()
240        args.version = 'pull-' + args.version
241    print(args.commit)
242    subprocess.check_call(['git', 'fetch'])
243    subprocess.check_call(['git', 'checkout', args.commit])
244    os.chdir(workdir)
245
246    os.chdir('gitian-builder')
247    subprocess.check_call(['git', 'pull'])
248    os.chdir(workdir)
249
250    if args.build:
251        build()
252
253    if args.sign:
254        sign()
255
256    if args.verify:
257        os.chdir('gitian.sigs')
258        subprocess.check_call(['git', 'pull'])
259        os.chdir(workdir)
260        sys.exit(verify())
261
262if __name__ == '__main__':
263    main()
264