1#!/usr/bin/env python3
2#
3# ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
4#
5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6# See https://llvm.org/LICENSE.txt for license information.
7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8#
9# ==-------------------------------------------------------------------------==#
10
11import argparse
12from git import Repo # type: ignore
13import github
14import os
15import re
16import requests
17import sys
18import time
19from typing import List, Optional
20
21class IssueSubscriber:
22
23    @property
24    def team_name(self) -> str:
25        return self._team_name
26
27    def __init__(self, token:str, repo:str, issue_number:int, label_name:str):
28        self.repo = github.Github(token).get_repo(repo)
29        self.org = github.Github(token).get_organization(self.repo.organization.login)
30        self.issue = self.repo.get_issue(issue_number)
31        self._team_name = 'issue-subscribers-{}'.format(label_name).lower()
32
33    def run(self) -> bool:
34        for team in self.org.get_teams():
35            if self.team_name != team.name.lower():
36                continue
37            comment = '@llvm/{}'.format(team.slug)
38            self.issue.create_comment(comment)
39            return True
40        return False
41
42def setup_llvmbot_git(git_dir = '.'):
43    """
44    Configure the git repo in `git_dir` with the llvmbot account so
45    commits are attributed to llvmbot.
46    """
47    repo = Repo(git_dir)
48    with repo.config_writer() as config:
49        config.set_value('user', 'name', 'llvmbot')
50        config.set_value('user', 'email', 'llvmbot@llvm.org')
51
52def phab_api_call(phab_token:str, url:str, args:dict) -> dict:
53    """
54    Make an API call to the Phabricator web service and return a dictionary
55    containing the json response.
56    """
57    data = { "api.token" : phab_token }
58    data.update(args)
59    response = requests.post(url, data = data)
60    return response.json()
61
62
63def phab_login_to_github_login(phab_token:str, repo:github.Repository.Repository, phab_login:str) -> Optional[str]:
64    """
65    Tries to translate a Phabricator login to a github login by
66    finding a commit made in Phabricator's Differential.
67    The commit's SHA1 is then looked up in the github repo and
68    the committer's login associated with that commit is returned.
69
70    :param str phab_token: The Conduit API token to use for communication with Pabricator
71    :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
72    :param str phab_login: The Phabricator login to be translated.
73    """
74
75    args = {
76        "constraints[authors][0]" : phab_login,
77        # PHID for "LLVM Github Monorepo" repository
78        "constraints[repositories][0]" : "PHID-REPO-f4scjekhnkmh7qilxlcy",
79        "limit" : 1
80    }
81    # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
82    r = phab_api_call(phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args)
83    data = r['result']['data']
84    if len(data) == 0:
85        # Can't find any commits associated with this user
86        return None
87
88    commit_sha = data[0]['fields']['identifier']
89    committer = repo.get_commit(commit_sha).committer
90    if not committer:
91        # This committer had an email address GitHub could not recognize, so
92        # it can't link the user to a GitHub account.
93        print(f"Warning: Can't find github account for {phab_login}")
94        return None
95    return committer.login
96
97def phab_get_commit_approvers(phab_token:str, commit:github.Commit.Commit) -> list:
98    args = { "corpus" : commit.commit.message }
99    # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
100    r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args)
101    review_id = r['result']['revisionIDFieldInfo']['value']
102    if not review_id:
103        # No Phabricator revision for this commit
104        return []
105
106    args = {
107        'constraints[ids][0]' : review_id,
108        'attachments[reviewers]' : True
109    }
110    # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
111    r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.revision.search", args)
112    reviewers = r['result']['data'][0]['attachments']['reviewers']['reviewers']
113    accepted = []
114    for reviewer in reviewers:
115        if reviewer['status'] != 'accepted':
116            continue
117        phid = reviewer['reviewerPHID']
118        args = { 'constraints[phids][0]' : phid }
119        # API documentation: https://reviews.llvm.org/conduit/method/user.search/
120        r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args)
121        accepted.append(r['result']['data'][0]['fields']['username'])
122    return accepted
123
124class ReleaseWorkflow:
125
126    CHERRY_PICK_FAILED_LABEL = 'release:cherry-pick-failed'
127
128    """
129    This class implements the sub-commands for the release-workflow command.
130    The current sub-commands are:
131        * create-branch
132        * create-pull-request
133
134    The execute_command method will automatically choose the correct sub-command
135    based on the text in stdin.
136    """
137
138    def __init__(self, token:str, repo:str, issue_number:int,
139                       branch_repo_name:str, branch_repo_token:str,
140                       llvm_project_dir:str, phab_token:str) -> None:
141        self._token = token
142        self._repo_name = repo
143        self._issue_number = issue_number
144        self._branch_repo_name = branch_repo_name
145        if branch_repo_token:
146            self._branch_repo_token = branch_repo_token
147        else:
148            self._branch_repo_token = self.token
149        self._llvm_project_dir = llvm_project_dir
150        self._phab_token = phab_token
151
152    @property
153    def token(self) -> str:
154        return self._token
155
156    @property
157    def repo_name(self) -> str:
158        return self._repo_name
159
160    @property
161    def issue_number(self) -> int:
162        return self._issue_number
163
164    @property
165    def branch_repo_name(self) -> str:
166        return self._branch_repo_name
167
168    @property
169    def branch_repo_token(self) -> str:
170        return self._branch_repo_token
171
172    @property
173    def llvm_project_dir(self) -> str:
174        return self._llvm_project_dir
175
176    @property
177    def phab_token(self) -> str:
178        return self._phab_token
179
180    @property
181    def repo(self) -> github.Repository.Repository:
182        return github.Github(self.token).get_repo(self.repo_name)
183
184    @property
185    def issue(self) -> github.Issue.Issue:
186        return self.repo.get_issue(self.issue_number)
187
188    @property
189    def push_url(self) -> str:
190        return 'https://{}@github.com/{}'.format(self.branch_repo_token, self.branch_repo_name)
191
192    @property
193    def branch_name(self) -> str:
194        return 'issue{}'.format(self.issue_number)
195
196    @property
197    def release_branch_for_issue(self) -> Optional[str]:
198        issue = self.issue
199        milestone = issue.milestone
200        if milestone is None:
201            return None
202        m = re.search('branch: (.+)',milestone.description)
203        if m:
204            return m.group(1)
205        return None
206
207    def print_release_branch(self) -> None:
208        print(self.release_branch_for_issue)
209
210    def issue_notify_branch(self) -> None:
211        self.issue.create_comment('/branch {}/{}'.format(self.branch_repo_name, self.branch_name))
212
213    def issue_notify_pull_request(self, pull:github.PullRequest.PullRequest) -> None:
214        self.issue.create_comment('/pull-request {}#{}'.format(self.branch_repo_name, pull.number))
215
216    def make_ignore_comment(self, comment: str) -> str:
217        """
218        Returns the comment string with a prefix that will cause
219        a Github workflow to skip parsing this comment.
220
221        :param str comment: The comment to ignore
222        """
223        return "<!--IGNORE-->\n"+comment
224
225    def issue_notify_no_milestone(self, comment:List[str]) -> None:
226        message = "{}\n\nError: Command failed due to missing milestone.".format(''.join(['>' + line for line in comment]))
227        self.issue.create_comment(self.make_ignore_comment(message))
228
229    @property
230    def action_url(self) -> str:
231        if os.getenv('CI'):
232            return 'https://github.com/{}/actions/runs/{}'.format(os.getenv('GITHUB_REPOSITORY'), os.getenv('GITHUB_RUN_ID'))
233        return ""
234
235    def issue_notify_cherry_pick_failure(self, commit:str) -> github.IssueComment.IssueComment:
236        message = self.make_ignore_comment("Failed to cherry-pick: {}\n\n".format(commit))
237        action_url = self.action_url
238        if action_url:
239            message += action_url + "\n\n"
240        message += "Please manually backport the fix and push it to your github fork.  Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
241        issue = self.issue
242        comment = issue.create_comment(message)
243        issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL)
244        return comment
245
246    def issue_notify_pull_request_failure(self, branch:str) -> github.IssueComment.IssueComment:
247        message = "Failed to create pull request for {} ".format(branch)
248        message += self.action_url
249        return self.issue.create_comment(message)
250
251    def issue_remove_cherry_pick_failed_label(self):
252        if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]:
253            self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL)
254
255    def pr_request_review(self, pr:github.PullRequest.PullRequest):
256        """
257        This function will try to find the best reviewers for `commits` and
258        then add a comment requesting review of the backport and assign the
259        pull request to the selected reviewers.
260
261        The reviewers selected are those users who approved the patch in
262        Phabricator.
263        """
264        reviewers = []
265        for commit in pr.get_commits():
266            approvers = phab_get_commit_approvers(self.phab_token, commit)
267            for a in approvers:
268                login = phab_login_to_github_login(self.phab_token, self.repo, a)
269                if not login:
270                    continue
271                reviewers.append(login)
272        if len(reviewers):
273            message = "{} What do you think about merging this PR to the release branch?".format(
274                    " ".join(["@" + r for r in reviewers]))
275            pr.create_issue_comment(message)
276            pr.add_to_assignees(*reviewers)
277
278    def create_branch(self, commits:List[str]) -> bool:
279        """
280        This function attempts to backport `commits` into the branch associated
281        with `self.issue_number`.
282
283        If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
284        a comment is added to the issue saying that the cherry-pick failed.
285
286        :param list commits: List of commits to cherry-pick.
287
288        """
289        print('cherry-picking', commits)
290        branch_name = self.branch_name
291        local_repo = Repo(self.llvm_project_dir)
292        local_repo.git.checkout(self.release_branch_for_issue)
293
294        for c in commits:
295            try:
296                local_repo.git.cherry_pick('-x', c)
297            except Exception as e:
298                self.issue_notify_cherry_pick_failure(c)
299                raise e
300
301        push_url = self.push_url
302        print('Pushing to {} {}'.format(push_url, branch_name))
303        local_repo.git.push(push_url, 'HEAD:{}'.format(branch_name), force=True)
304
305        self.issue_notify_branch()
306        self.issue_remove_cherry_pick_failed_label()
307        return True
308
309    def check_if_pull_request_exists(self, repo:github.Repository.Repository, head:str) -> bool:
310        pulls = repo.get_pulls(head=head)
311        return pulls.totalCount != 0
312
313    def create_pull_request(self, owner:str, repo_name:str, branch:str) -> bool:
314        """
315        reate a pull request in `self.branch_repo_name`.  The base branch of the
316        pull request will be chosen based on the the milestone attached to
317        the issue represented by `self.issue_number`  For example if the milestone
318        is Release 13.0.1, then the base branch will be release/13.x. `branch`
319        will be used as the compare branch.
320        https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
321        https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
322        """
323        repo = github.Github(self.token).get_repo(self.branch_repo_name)
324        issue_ref = '{}#{}'.format(self.repo_name, self.issue_number)
325        pull = None
326        release_branch_for_issue = self.release_branch_for_issue
327        if release_branch_for_issue is None:
328            return False
329        head_branch = branch
330        if not repo.fork:
331            # If the target repo is not a fork of llvm-project, we need to copy
332            # the branch into the target repo.  GitHub only supports cross-repo pull
333            # requests on forked repos.
334            head_branch = f'{owner}-{branch}'
335            local_repo = Repo(self.llvm_project_dir)
336            push_done = False
337            for _ in range(0,5):
338                try:
339                    local_repo.git.fetch(f'https://github.com/{owner}/{repo_name}', f'{branch}:{branch}')
340                    local_repo.git.push(self.push_url, f'{branch}:{head_branch}', force=True)
341                    push_done = True
342                    break
343                except Exception as e:
344                    print(e)
345                    time.sleep(30)
346                    continue
347            if not push_done:
348                raise Exception("Failed to mirror branch into {}".format(self.push_url))
349            owner = repo.owner.login
350
351        head = f"{owner}:{head_branch}"
352        if self.check_if_pull_request_exists(repo, head):
353            print("PR already exists...")
354            return True
355        try:
356            pull = repo.create_pull(title=f"PR for {issue_ref}",
357                                    body='resolves {}'.format(issue_ref),
358                                    base=release_branch_for_issue,
359                                    head=head,
360                                    maintainer_can_modify=False)
361
362            try:
363                if self.phab_token:
364                    self.pr_request_review(pull)
365            except Exception as e:
366                print("error: Failed while searching for reviewers", e)
367
368        except Exception as e:
369            self.issue_notify_pull_request_failure(branch)
370            raise e
371
372        if pull is None:
373            return False
374
375        self.issue_notify_pull_request(pull)
376        self.issue_remove_cherry_pick_failed_label()
377
378        # TODO(tstellar): Do you really want to always return True?
379        return True
380
381
382    def execute_command(self) -> bool:
383        """
384        This function reads lines from STDIN and executes the first command
385        that it finds.  The 2 supported commands are:
386        /cherry-pick commit0 <commit1> <commit2> <...>
387        /branch <owner>/<repo>/<branch>
388        """
389        for line in sys.stdin:
390            line.rstrip()
391            m = re.search(r"/([a-z-]+)\s(.+)", line)
392            if not m:
393                continue
394            command = m.group(1)
395            args = m.group(2)
396
397            if command == 'cherry-pick':
398                return self.create_branch(args.split())
399
400            if command == 'branch':
401                m = re.match('([^/]+)/([^/]+)/(.+)', args)
402                if m:
403                    owner = m.group(1)
404                    repo = m.group(2)
405                    branch = m.group(3)
406                    return self.create_pull_request(owner, repo, branch)
407
408        print("Do not understand input:")
409        print(sys.stdin.readlines())
410        return False
411
412parser = argparse.ArgumentParser()
413parser.add_argument('--token', type=str, required=True, help='GitHub authentiation token')
414parser.add_argument('--repo', type=str, default=os.getenv('GITHUB_REPOSITORY', 'llvm/llvm-project'),
415                    help='The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)')
416subparsers = parser.add_subparsers(dest='command')
417
418issue_subscriber_parser = subparsers.add_parser('issue-subscriber')
419issue_subscriber_parser.add_argument('--label-name', type=str, required=True)
420issue_subscriber_parser.add_argument('--issue-number', type=int, required=True)
421
422release_workflow_parser = subparsers.add_parser('release-workflow')
423release_workflow_parser.add_argument('--llvm-project-dir', type=str, default='.', help='directory containing the llvm-project checout')
424release_workflow_parser.add_argument('--issue-number', type=int, required=True, help='The issue number to update')
425release_workflow_parser.add_argument('--phab-token', type=str, help='Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/')
426release_workflow_parser.add_argument('--branch-repo-token', type=str,
427                                     help='GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.')
428release_workflow_parser.add_argument('--branch-repo', type=str, default='llvm/llvm-project-release-prs',
429                                     help='The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)')
430release_workflow_parser.add_argument('sub_command', type=str, choices=['print-release-branch', 'auto'],
431                                     help='Print to stdout the name of the release branch ISSUE_NUMBER should be backported to')
432
433llvmbot_git_config_parser = subparsers.add_parser('setup-llvmbot-git', help='Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot')
434
435args = parser.parse_args()
436
437if args.command == 'issue-subscriber':
438    issue_subscriber = IssueSubscriber(args.token, args.repo, args.issue_number, args.label_name)
439    issue_subscriber.run()
440elif args.command == 'release-workflow':
441    release_workflow = ReleaseWorkflow(args.token, args.repo, args.issue_number,
442                                       args.branch_repo, args.branch_repo_token,
443                                       args.llvm_project_dir, args.phab_token)
444    if not release_workflow.release_branch_for_issue:
445        release_workflow.issue_notify_no_milestone(sys.stdin.readlines())
446        sys.exit(1)
447    if args.sub_command == 'print-release-branch':
448        release_workflow.print_release_branch()
449    else:
450        if not release_workflow.execute_command():
451            sys.exit(1)
452elif args.command == 'setup-llvmbot-git':
453    setup_llvmbot_git()
454