1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, # You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, unicode_literals 6 7import os 8import re 9import subprocess 10import sys 11 12import logging 13 14from mach.decorators import ( 15 CommandArgument, 16 Command, 17) 18 19import mozpack.path as mozpath 20 21import json 22 23GITHUB_ROOT = "https://github.com/" 24PR_REPOSITORIES = { 25 "webrender": { 26 "github": "servo/webrender", 27 "path": "gfx/wr", 28 "bugzilla_product": "Core", 29 "bugzilla_component": "Graphics: WebRender", 30 }, 31 "webgpu": { 32 "github": "gfx-rs/wgpu", 33 "path": "gfx/wgpu", 34 "bugzilla_product": "Core", 35 "bugzilla_component": "Graphics: WebGPU", 36 }, 37 "debugger": { 38 "github": "firefox-devtools/debugger", 39 "path": "devtools/client/debugger", 40 "bugzilla_product": "DevTools", 41 "bugzilla_component": "Debugger", 42 }, 43} 44 45 46@Command( 47 "import-pr", 48 category="misc", 49 description="Import a pull request from Github to the local repo.", 50) 51@CommandArgument("-b", "--bug-number", help="Bug number to use in the commit messages.") 52@CommandArgument( 53 "-t", 54 "--bugzilla-token", 55 help="Bugzilla API token used to file a new bug if no bug number is provided.", 56) 57@CommandArgument("-r", "--reviewer", help="Reviewer nick to apply to commit messages.") 58@CommandArgument( 59 "pull_request", 60 help="URL to the pull request to import (e.g. " 61 "https://github.com/servo/webrender/pull/3665).", 62) 63def import_pr( 64 command_context, 65 pull_request, 66 bug_number=None, 67 bugzilla_token=None, 68 reviewer=None, 69): 70 import requests 71 72 pr_number = None 73 repository = None 74 for r in PR_REPOSITORIES.values(): 75 if pull_request.startswith(GITHUB_ROOT + r["github"] + "/pull/"): 76 # sanitize URL, dropping anything after the PR number 77 pr_number = int(re.search("/pull/([0-9]+)", pull_request).group(1)) 78 pull_request = GITHUB_ROOT + r["github"] + "/pull/" + str(pr_number) 79 repository = r 80 break 81 82 if repository is None: 83 command_context.log( 84 logging.ERROR, 85 "unrecognized_repo", 86 {}, 87 "The pull request URL was not recognized; add it to the list of " 88 "recognized repos in PR_REPOSITORIES in %s" % __file__, 89 ) 90 sys.exit(1) 91 92 command_context.log( 93 logging.INFO, 94 "import_pr", 95 {"pr_url": pull_request}, 96 "Attempting to import {pr_url}", 97 ) 98 dirty = [ 99 f 100 for f in command_context.repository.get_changed_files(mode="all") 101 if f.startswith(repository["path"]) 102 ] 103 if dirty: 104 command_context.log( 105 logging.ERROR, 106 "dirty_tree", 107 repository, 108 "Local {path} tree is dirty; aborting!", 109 ) 110 sys.exit(1) 111 target_dir = mozpath.join( 112 command_context.topsrcdir, os.path.normpath(repository["path"]) 113 ) 114 115 if bug_number is None: 116 if bugzilla_token is None: 117 command_context.log( 118 logging.WARNING, 119 "no_token", 120 {}, 121 "No bug number or bugzilla API token provided; bug number will not " 122 "be added to commit messages.", 123 ) 124 else: 125 bug_number = _file_bug( 126 command_context, bugzilla_token, repository, pr_number 127 ) 128 elif bugzilla_token is not None: 129 command_context.log( 130 logging.WARNING, 131 "too_much_bug", 132 {}, 133 "Providing a bugzilla token is unnecessary when a bug number is provided. " 134 "Using bug number; ignoring token.", 135 ) 136 137 pr_patch = requests.get(pull_request + ".patch") 138 pr_patch.raise_for_status() 139 for patch in _split_patches(pr_patch.content, bug_number, pull_request, reviewer): 140 command_context.log( 141 logging.INFO, 142 "commit_msg", 143 patch, 144 "Processing commit [{commit_summary}] by [{author}] at [{date}]", 145 ) 146 patch_cmd = subprocess.Popen( 147 ["patch", "-p1", "-s"], stdin=subprocess.PIPE, cwd=target_dir 148 ) 149 patch_cmd.stdin.write(patch["diff"].encode("utf-8")) 150 patch_cmd.stdin.close() 151 patch_cmd.wait() 152 if patch_cmd.returncode != 0: 153 command_context.log( 154 logging.ERROR, 155 "commit_fail", 156 {}, 157 'Error applying diff from commit via "patch -p1 -s". Aborting...', 158 ) 159 sys.exit(patch_cmd.returncode) 160 command_context.repository.commit( 161 patch["commit_msg"], patch["author"], patch["date"], [target_dir] 162 ) 163 command_context.log(logging.INFO, "commit_pass", {}, "Committed successfully.") 164 165 166def _file_bug(command_context, token, repo, pr_number): 167 import requests 168 169 bug = requests.post( 170 "https://bugzilla.mozilla.org/rest/bug?api_key=%s" % token, 171 json={ 172 "product": repo["bugzilla_product"], 173 "component": repo["bugzilla_component"], 174 "summary": "Land %s#%s in mozilla-central" % (repo["github"], pr_number), 175 "version": "unspecified", 176 }, 177 ) 178 bug.raise_for_status() 179 command_context.log(logging.DEBUG, "new_bug", {}, bug.content) 180 bugnumber = json.loads(bug.content)["id"] 181 command_context.log( 182 logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}" 183 ) 184 return bugnumber 185 186 187def _split_patches(patchfile, bug_number, pull_request, reviewer): 188 INITIAL = 0 189 HEADERS = 1 190 STAT_AND_DIFF = 2 191 192 patch = b"" 193 state = INITIAL 194 for line in patchfile.splitlines(): 195 if state == INITIAL: 196 if line.startswith(b"From "): 197 state = HEADERS 198 elif state == HEADERS: 199 patch += line + b"\n" 200 if line == b"---": 201 state = STAT_AND_DIFF 202 elif state == STAT_AND_DIFF: 203 if line.startswith(b"From "): 204 yield _parse_patch(patch, bug_number, pull_request, reviewer) 205 patch = b"" 206 state = HEADERS 207 else: 208 patch += line + b"\n" 209 if len(patch) > 0: 210 yield _parse_patch(patch, bug_number, pull_request, reviewer) 211 return 212 213 214def _parse_patch(patch, bug_number, pull_request, reviewer): 215 import email 216 from email import ( 217 header, 218 policy, 219 ) 220 221 parse_policy = policy.compat32.clone(max_line_length=None) 222 parsed_mail = email.message_from_bytes(patch, policy=parse_policy) 223 224 def header_as_unicode(key): 225 decoded = header.decode_header(parsed_mail[key]) 226 return str(header.make_header(decoded)) 227 228 author = header_as_unicode("From") 229 date = header_as_unicode("Date") 230 commit_summary = header_as_unicode("Subject") 231 email_body = parsed_mail.get_payload(decode=True).decode("utf-8") 232 (commit_body, diff) = ("\n" + email_body).rsplit("\n---\n", 1) 233 234 bug_prefix = "" 235 if bug_number is not None: 236 bug_prefix = "Bug %s - " % bug_number 237 commit_summary = re.sub(r"^\[PATCH[0-9 /]*\] ", bug_prefix, commit_summary) 238 if reviewer is not None: 239 commit_summary += " r=" + reviewer 240 241 commit_msg = commit_summary + "\n" 242 if len(commit_body) > 0: 243 commit_msg += commit_body + "\n" 244 commit_msg += "\n[import_pr] From " + pull_request + "\n" 245 246 patch_obj = { 247 "author": author, 248 "date": date, 249 "commit_summary": commit_summary, 250 "commit_msg": commit_msg, 251 "diff": diff, 252 } 253 return patch_obj 254