1#!/usr/bin/env python3
2# Copyright (c) 2016-2017 Bitcoin Core Developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6# This script will locally construct a merge commit for a pull request on a
7# github repository, inspect it, sign it and optionally push it.
8
9# The following temporary branches are created/overwritten and deleted:
10# * pull/$PULL/base (the current master we're merging onto)
11# * pull/$PULL/head (the current state of the remote pull request)
12# * pull/$PULL/merge (github's merge)
13# * pull/$PULL/local-merge (our merge)
14
15# In case of a clean merge that is accepted by the user, the local branch with
16# name $BRANCH is overwritten with the merged result, and optionally pushed.
17import os
18from sys import stdin,stdout,stderr
19import argparse
20import hashlib
21import subprocess
22import sys
23import json
24import codecs
25from urllib.request import Request, urlopen
26from urllib.error import HTTPError
27
28# External tools (can be overridden using environment)
29GIT = os.getenv('GIT','git')
30BASH = os.getenv('BASH','bash')
31
32# OS specific configuration for terminal attributes
33ATTR_RESET = ''
34ATTR_PR = ''
35COMMIT_FORMAT = '%h %s (%an)%d'
36if os.name == 'posix': # if posix, assume we can use basic terminal escapes
37    ATTR_RESET = '\033[0m'
38    ATTR_PR = '\033[1;36m'
39    COMMIT_FORMAT = '%C(bold blue)%h%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset'
40
41def git_config_get(option, default=None):
42    '''
43    Get named configuration option from git repository.
44    '''
45    try:
46        return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8')
47    except subprocess.CalledProcessError:
48        return default
49
50def retrieve_pr_info(repo,pull,ghtoken):
51    '''
52    Retrieve pull request information from github.
53    Return None if no title can be found, or an error happens.
54    '''
55    try:
56        req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull)
57        if ghtoken is not None:
58            req.add_header('Authorization', 'token ' + ghtoken)
59        result = urlopen(req)
60        reader = codecs.getreader('utf-8')
61        obj = json.load(reader(result))
62        return obj
63    except HTTPError as e:
64        error_message = e.read()
65        print('Warning: unable to retrieve pull information from github: %s' % e)
66        print('Detailed error: %s' % error_message)
67        return None
68    except Exception as e:
69        print('Warning: unable to retrieve pull information from github: %s' % e)
70        return None
71
72def ask_prompt(text):
73    print(text,end=" ",file=stderr)
74    stderr.flush()
75    reply = stdin.readline().rstrip()
76    print("",file=stderr)
77    return reply
78
79def get_symlink_files():
80    files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines())
81    ret = []
82    for f in files:
83        if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000:
84            ret.append(f.decode('utf-8').split("\t")[1])
85    return ret
86
87def tree_sha512sum(commit='HEAD'):
88    # request metadata for entire tree, recursively
89    files = []
90    blob_by_name = {}
91    for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
92        name_sep = line.index(b'\t')
93        metadata = line[:name_sep].split() # perms, 'blob', blobid
94        assert(metadata[1] == b'blob')
95        name = line[name_sep+1:]
96        files.append(name)
97        blob_by_name[name] = metadata[2]
98
99    files.sort()
100    # open connection to git-cat-file in batch mode to request data for all blobs
101    # this is much faster than launching it per file
102    p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
103    overall = hashlib.sha512()
104    for f in files:
105        blob = blob_by_name[f]
106        # request blob
107        p.stdin.write(blob + b'\n')
108        p.stdin.flush()
109        # read header: blob, "blob", size
110        reply = p.stdout.readline().split()
111        assert(reply[0] == blob and reply[1] == b'blob')
112        size = int(reply[2])
113        # hash the blob data
114        intern = hashlib.sha512()
115        ptr = 0
116        while ptr < size:
117            bs = min(65536, size - ptr)
118            piece = p.stdout.read(bs)
119            if len(piece) == bs:
120                intern.update(piece)
121            else:
122                raise IOError('Premature EOF reading git cat-file output')
123            ptr += bs
124        dig = intern.hexdigest()
125        assert(p.stdout.read(1) == b'\n') # ignore LF that follows blob data
126        # update overall hash with file hash
127        overall.update(dig.encode("utf-8"))
128        overall.update("  ".encode("utf-8"))
129        overall.update(f)
130        overall.update("\n".encode("utf-8"))
131    p.stdin.close()
132    if p.wait():
133        raise IOError('Non-zero return value executing git cat-file')
134    return overall.hexdigest()
135
136def print_merge_details(pull, title, branch, base_branch, head_branch):
137    print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET))
138    subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch])
139
140def parse_arguments():
141    epilog = '''
142        In addition, you can set the following git configuration variables:
143        githubmerge.repository (mandatory),
144        user.signingkey (mandatory),
145        user.ghtoken (default: none).
146        githubmerge.host (default: git@github.com),
147        githubmerge.branch (no default),
148        githubmerge.testcmd (default: none).
149    '''
150    parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests',
151            epilog=epilog)
152    parser.add_argument('pull', metavar='PULL', type=int, nargs=1,
153        help='Pull request ID to merge')
154    parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?',
155        default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')')
156    return parser.parse_args()
157
158def main():
159    # Extract settings from git repo
160    repo = git_config_get('githubmerge.repository')
161    host = git_config_get('githubmerge.host','git@github.com')
162    opt_branch = git_config_get('githubmerge.branch',None)
163    testcmd = git_config_get('githubmerge.testcmd')
164    ghtoken = git_config_get('user.ghtoken')
165    signingkey = git_config_get('user.signingkey')
166    if repo is None:
167        print("ERROR: No repository configured. Use this command to set:", file=stderr)
168        print("git config githubmerge.repository <owner>/<repo>", file=stderr)
169        sys.exit(1)
170    if signingkey is None:
171        print("ERROR: No GPG signing key set. Set one using:",file=stderr)
172        print("git config --global user.signingkey <key>",file=stderr)
173        sys.exit(1)
174
175    if host.startswith(('https:','http:')):
176        host_repo = host+"/"+repo+".git"
177    else:
178        host_repo = host+":"+repo
179
180    # Extract settings from command line
181    args = parse_arguments()
182    pull = str(args.pull[0])
183
184    # Receive pull information from github
185    info = retrieve_pr_info(repo,pull,ghtoken)
186    if info is None:
187        sys.exit(1)
188    title = info['title'].strip()
189    body = info['body'].strip()
190    # precedence order for destination branch argument:
191    #   - command line argument
192    #   - githubmerge.branch setting
193    #   - base branch for pull (as retrieved from github)
194    #   - 'master'
195    branch = args.branch or opt_branch or info['base']['ref'] or 'master'
196
197    # Initialize source branches
198    head_branch = 'pull/'+pull+'/head'
199    base_branch = 'pull/'+pull+'/base'
200    merge_branch = 'pull/'+pull+'/merge'
201    local_merge_branch = 'pull/'+pull+'/local-merge'
202
203    devnull = open(os.devnull, 'w', encoding="utf8")
204    try:
205        subprocess.check_call([GIT,'checkout','-q',branch])
206    except subprocess.CalledProcessError:
207        print("ERROR: Cannot check out branch %s." % (branch), file=stderr)
208        sys.exit(3)
209    try:
210        subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*',
211                                                          '+refs/heads/'+branch+':refs/heads/'+base_branch])
212    except subprocess.CalledProcessError:
213        print("ERROR: Cannot find pull request #%s or branch %s on %s." % (pull,branch,host_repo), file=stderr)
214        sys.exit(3)
215    try:
216        subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout)
217    except subprocess.CalledProcessError:
218        print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr)
219        sys.exit(3)
220    try:
221        subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout)
222    except subprocess.CalledProcessError:
223        print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr)
224        sys.exit(3)
225    subprocess.check_call([GIT,'checkout','-q',base_branch])
226    subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull)
227    subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch])
228
229    try:
230        # Go up to the repository's root.
231        toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']).strip()
232        os.chdir(toplevel)
233        # Create unsigned merge commit.
234        if title:
235            firstline = 'Merge #%s: %s' % (pull,title)
236        else:
237            firstline = 'Merge #%s' % (pull,)
238        message = firstline + '\n\n'
239        message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8')
240        message += '\n\nPull request description:\n\n  ' + body.replace('\n', '\n  ') + '\n'
241        try:
242            subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch])
243        except subprocess.CalledProcessError:
244            print("ERROR: Cannot be merged cleanly.",file=stderr)
245            subprocess.check_call([GIT,'merge','--abort'])
246            sys.exit(4)
247        logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8')
248        if logmsg.rstrip() != firstline.rstrip():
249            print("ERROR: Creating merge failed (already merged?).",file=stderr)
250            sys.exit(4)
251
252        symlink_files = get_symlink_files()
253        for f in symlink_files:
254            print("ERROR: File %s was a symlink" % f)
255        if len(symlink_files) > 0:
256            sys.exit(4)
257
258        # Put tree SHA512 into the message
259        try:
260            first_sha512 = tree_sha512sum()
261            message += '\n\nTree-SHA512: ' + first_sha512
262        except subprocess.CalledProcessError:
263            print("ERROR: Unable to compute tree hash")
264            sys.exit(4)
265        try:
266            subprocess.check_call([GIT,'commit','--amend','-m',message.encode('utf-8')])
267        except subprocess.CalledProcessError:
268            print("ERROR: Cannot update message.", file=stderr)
269            sys.exit(4)
270
271        print_merge_details(pull, title, branch, base_branch, head_branch)
272        print()
273
274        # Run test command if configured.
275        if testcmd:
276            if subprocess.call(testcmd,shell=True):
277                print("ERROR: Running %s failed." % testcmd,file=stderr)
278                sys.exit(5)
279
280            # Show the created merge.
281            diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch])
282            subprocess.check_call([GIT,'diff',base_branch+'..'+local_merge_branch])
283            if diff:
284                print("WARNING: merge differs from github!",file=stderr)
285                reply = ask_prompt("Type 'ignore' to continue.")
286                if reply.lower() == 'ignore':
287                    print("Difference with github ignored.",file=stderr)
288                else:
289                    sys.exit(6)
290        else:
291            # Verify the result manually.
292            print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr)
293            print("Run 'git diff HEAD~' to show the changes being merged.",file=stderr)
294            print("Type 'exit' when done.",file=stderr)
295            if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt
296                os.putenv('debian_chroot',pull)
297            subprocess.call([BASH,'-i'])
298
299        second_sha512 = tree_sha512sum()
300        if first_sha512 != second_sha512:
301            print("ERROR: Tree hash changed unexpectedly",file=stderr)
302            sys.exit(8)
303
304        # Sign the merge commit.
305        print_merge_details(pull, title, branch, base_branch, head_branch)
306        while True:
307            reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower()
308            if reply == 's':
309                try:
310                    subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit'])
311                    break
312                except subprocess.CalledProcessError:
313                    print("Error while signing, asking again.",file=stderr)
314            elif reply == 'x':
315                print("Not signing off on merge, exiting.",file=stderr)
316                sys.exit(1)
317
318        # Put the result in branch.
319        subprocess.check_call([GIT,'checkout','-q',branch])
320        subprocess.check_call([GIT,'reset','-q','--hard',local_merge_branch])
321    finally:
322        # Clean up temporary branches.
323        subprocess.call([GIT,'checkout','-q',branch])
324        subprocess.call([GIT,'branch','-q','-D',head_branch],stderr=devnull)
325        subprocess.call([GIT,'branch','-q','-D',base_branch],stderr=devnull)
326        subprocess.call([GIT,'branch','-q','-D',merge_branch],stderr=devnull)
327        subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull)
328
329    # Push the result.
330    while True:
331        reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower()
332        if reply == 'push':
333            subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch])
334            break
335        elif reply == 'x':
336            sys.exit(1)
337
338if __name__ == '__main__':
339    main()
340
341