1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7""" l10n_bumper.py 8 9 Updates a gecko repo with up to date changesets from l10n.mozilla.org. 10 11 Specifically, it updates l10n-changesets.json which is used by mobile releases. 12 13 This is to allow for `mach taskgraph` to reference specific l10n revisions 14 without having to resort to task.extra or commandline base64 json hacks. 15""" 16from __future__ import absolute_import 17import codecs 18import os 19import pprint 20import sys 21import time 22 23try: 24 import simplejson as json 25 26 assert json 27except ImportError: 28 import json 29 30sys.path.insert(1, os.path.dirname(sys.path[0])) 31 32from mozharness.base.errors import HgErrorList 33from mozharness.base.vcs.vcsbase import VCSScript 34from mozharness.base.log import FATAL 35 36 37class L10nBumper(VCSScript): 38 config_options = [ 39 [ 40 [ 41 "--ignore-closed-tree", 42 ], 43 { 44 "action": "store_true", 45 "dest": "ignore_closed_tree", 46 "default": False, 47 "help": "Bump l10n changesets on a closed tree.", 48 }, 49 ], 50 [ 51 [ 52 "--build", 53 ], 54 { 55 "action": "store_false", 56 "dest": "dontbuild", 57 "default": True, 58 "help": "Trigger new builds on push.", 59 }, 60 ], 61 ] 62 63 def __init__(self, require_config_file=True): 64 super(L10nBumper, self).__init__( 65 all_actions=[ 66 "clobber", 67 "check-treestatus", 68 "checkout-gecko", 69 "bump-changesets", 70 "push", 71 "push-loop", 72 ], 73 default_actions=[ 74 "push-loop", 75 ], 76 require_config_file=require_config_file, 77 config_options=self.config_options, 78 # Default config options 79 config={ 80 "treestatus_base_url": "https://treestatus.mozilla-releng.net", 81 "log_max_rotate": 99, 82 }, 83 ) 84 85 # Helper methods {{{1 86 def query_abs_dirs(self): 87 if self.abs_dirs: 88 return self.abs_dirs 89 90 abs_dirs = super(L10nBumper, self).query_abs_dirs() 91 92 abs_dirs.update( 93 { 94 "gecko_local_dir": os.path.join( 95 abs_dirs["abs_work_dir"], 96 self.config.get( 97 "gecko_local_dir", 98 os.path.basename(self.config["gecko_pull_url"]), 99 ), 100 ), 101 } 102 ) 103 self.abs_dirs = abs_dirs 104 return self.abs_dirs 105 106 def hg_commit(self, path, repo_path, message): 107 """ 108 Commits changes in repo_path, with specified user and commit message 109 """ 110 user = self.config["hg_user"] 111 hg = self.query_exe("hg", return_type="list") 112 env = self.query_env(partial_env={"LANG": "en_US.UTF-8"}) 113 cmd = hg + ["add", path] 114 self.run_command(cmd, cwd=repo_path, env=env) 115 cmd = hg + ["commit", "-u", user, "-m", message] 116 self.run_command(cmd, cwd=repo_path, env=env) 117 118 def hg_push(self, repo_path): 119 hg = self.query_exe("hg", return_type="list") 120 command = hg + [ 121 "push", 122 "-e", 123 "ssh -oIdentityFile=%s -l %s" 124 % ( 125 self.config["ssh_key"], 126 self.config["ssh_user"], 127 ), 128 "-r", 129 ".", 130 self.config["gecko_push_url"], 131 ] 132 status = self.run_command(command, cwd=repo_path, error_list=HgErrorList) 133 if status != 0: 134 # We failed; get back to a known state so we can either retry 135 # or fail out and continue later. 136 self.run_command( 137 hg 138 + ["--config", "extensions.mq=", "strip", "--no-backup", "outgoing()"], 139 cwd=repo_path, 140 ) 141 self.run_command(hg + ["up", "-C"], cwd=repo_path) 142 self.run_command( 143 hg + ["--config", "extensions.purge=", "purge", "--all"], cwd=repo_path 144 ) 145 return False 146 return True 147 148 def _read_json(self, path): 149 contents = self.read_from_file(path) 150 try: 151 json_contents = json.loads(contents) 152 return json_contents 153 except ValueError: 154 self.error("%s is invalid json!" % path) 155 156 def _read_version(self, path): 157 contents = self.read_from_file(path).split("\n")[0] 158 return contents.split(".") 159 160 def _build_locale_map(self, old_contents, new_contents): 161 locale_map = {} 162 for key in old_contents: 163 if key not in new_contents: 164 locale_map[key] = "removed" 165 for k, v in new_contents.items(): 166 if old_contents.get(k, {}).get("revision") != v["revision"]: 167 locale_map[k] = v["revision"] 168 elif old_contents.get(k, {}).get("platforms") != v["platforms"]: 169 locale_map[k] = v["platforms"] 170 return locale_map 171 172 def _build_platform_dict(self, bump_config): 173 dirs = self.query_abs_dirs() 174 repo_path = dirs["gecko_local_dir"] 175 platform_dict = {} 176 ignore_config = bump_config.get("ignore_config", {}) 177 for platform_config in bump_config["platform_configs"]: 178 path = os.path.join(repo_path, platform_config["path"]) 179 self.info( 180 "Reading %s for %s locales..." % (path, platform_config["platforms"]) 181 ) 182 contents = self.read_from_file(path) 183 for locale in contents.splitlines(): 184 # locale is 1st word in line in shipped-locales 185 if platform_config.get("format") == "shipped-locales": 186 locale = locale.split(" ")[0] 187 existing_platforms = set( 188 platform_dict.get(locale, {}).get("platforms", []) 189 ) 190 platforms = set(platform_config["platforms"]) 191 ignore_platforms = set(ignore_config.get(locale, [])) 192 platforms = (platforms | existing_platforms) - ignore_platforms 193 platform_dict[locale] = {"platforms": sorted(list(platforms))} 194 self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict)) 195 return platform_dict 196 197 def _build_revision_dict(self, bump_config, version_list): 198 self.info("Building revision dict...") 199 platform_dict = self._build_platform_dict(bump_config) 200 revision_dict = {} 201 if bump_config.get("revision_url"): 202 repl_dict = { 203 "MAJOR_VERSION": version_list[0], 204 "COMBINED_MAJOR_VERSION": str( 205 int(version_list[0]) + int(version_list[1]) 206 ), 207 } 208 209 url = bump_config["revision_url"] % repl_dict 210 path = self.download_file(url, error_level=FATAL) 211 revision_info = self.read_from_file(path) 212 self.info("Got %s" % revision_info) 213 for line in revision_info.splitlines(): 214 locale, revision = line.split(" ") 215 if locale in platform_dict: 216 revision_dict[locale] = platform_dict[locale] 217 revision_dict[locale]["revision"] = revision 218 else: 219 for k, v in platform_dict.items(): 220 v["revision"] = "default" 221 revision_dict[k] = v 222 self.info("revision_dict:\n%s" % pprint.pformat(revision_dict)) 223 return revision_dict 224 225 def build_commit_message(self, name, locale_map): 226 comments = "" 227 approval_str = "r=release a=l10n-bump" 228 for locale, revision in sorted(locale_map.items()): 229 comments += "%s -> %s\n" % (locale, revision) 230 if self.config["dontbuild"]: 231 approval_str += " DONTBUILD" 232 if self.config["ignore_closed_tree"]: 233 approval_str += " CLOSED TREE" 234 message = "no bug - Bumping %s %s\n\n" % (name, approval_str) 235 message += comments 236 message = message.encode("utf-8") 237 return message 238 239 def query_treestatus(self): 240 "Return True if we can land based on treestatus" 241 c = self.config 242 dirs = self.query_abs_dirs() 243 tree = c.get( 244 "treestatus_tree", os.path.basename(c["gecko_pull_url"].rstrip("/")) 245 ) 246 treestatus_url = "%s/trees/%s" % (c["treestatus_base_url"], tree) 247 treestatus_json = os.path.join(dirs["abs_work_dir"], "treestatus.json") 248 if not os.path.exists(dirs["abs_work_dir"]): 249 self.mkdir_p(dirs["abs_work_dir"]) 250 self.rmtree(treestatus_json) 251 252 self.run_command( 253 ["curl", "--retry", "4", "-o", treestatus_json, treestatus_url], 254 throw_exception=True, 255 ) 256 257 treestatus = self._read_json(treestatus_json) 258 if treestatus["result"]["status"] != "closed": 259 self.info( 260 "treestatus is %s - assuming we can land" 261 % repr(treestatus["result"]["status"]) 262 ) 263 return True 264 265 return False 266 267 # Actions {{{1 268 def check_treestatus(self): 269 if not self.config["ignore_closed_tree"] and not self.query_treestatus(): 270 self.info("breaking early since treestatus is closed") 271 sys.exit(0) 272 273 def checkout_gecko(self): 274 c = self.config 275 dirs = self.query_abs_dirs() 276 dest = dirs["gecko_local_dir"] 277 repos = [ 278 { 279 "repo": c["gecko_pull_url"], 280 "tag": c.get("gecko_tag", "default"), 281 "dest": dest, 282 "vcs": "hg", 283 } 284 ] 285 self.vcs_checkout_repos(repos) 286 287 def bump_changesets(self): 288 dirs = self.query_abs_dirs() 289 repo_path = dirs["gecko_local_dir"] 290 version_path = os.path.join(repo_path, self.config["version_path"]) 291 changes = False 292 version_list = self._read_version(version_path) 293 for bump_config in self.config["bump_configs"]: 294 path = os.path.join(repo_path, bump_config["path"]) 295 # For now, assume format == 'json'. When we add desktop support, 296 # we may need to add flatfile support 297 if os.path.exists(path): 298 old_contents = self._read_json(path) 299 else: 300 old_contents = {} 301 302 new_contents = self._build_revision_dict(bump_config, version_list) 303 304 if new_contents == old_contents: 305 continue 306 # super basic sanity check 307 if not isinstance(new_contents, dict) or len(new_contents) < 5: 308 self.error( 309 "Cowardly refusing to land a broken-seeming changesets file!" 310 ) 311 continue 312 313 # Write to disk 314 content_string = json.dumps( 315 new_contents, 316 sort_keys=True, 317 indent=4, 318 separators=(",", ": "), 319 ) 320 fh = codecs.open(path, encoding="utf-8", mode="w+") 321 fh.write(content_string + "\n") 322 fh.close() 323 324 locale_map = self._build_locale_map(old_contents, new_contents) 325 326 # Commit 327 message = self.build_commit_message(bump_config["name"], locale_map) 328 self.hg_commit(path, repo_path, message) 329 changes = True 330 return changes 331 332 def push(self): 333 dirs = self.query_abs_dirs() 334 repo_path = dirs["gecko_local_dir"] 335 return self.hg_push(repo_path) 336 337 def push_loop(self): 338 max_retries = 5 339 for _ in range(max_retries): 340 changed = False 341 if not self.config["ignore_closed_tree"] and not self.query_treestatus(): 342 # Tree is closed; exit early to avoid a bunch of wasted time 343 self.info("breaking early since treestatus is closed") 344 break 345 346 self.checkout_gecko() 347 if self.bump_changesets(): 348 changed = True 349 350 if not changed: 351 # Nothing changed, we're all done 352 self.info("No changes - all done") 353 break 354 355 if self.push(): 356 # We did it! Hurray! 357 self.info("Great success!") 358 break 359 # If we're here, then the push failed. It also stripped any 360 # outgoing commits, so we should be in a pristine state again 361 # Empty our local cache of manifests so they get loaded again next 362 # time through this loop. This makes sure we get fresh upstream 363 # manifests, and avoids problems like bug 979080 364 self.device_manifests = {} 365 366 # Sleep before trying again 367 self.info("Sleeping 60 before trying again") 368 time.sleep(60) 369 else: 370 self.fatal("Didn't complete successfully (hit max_retries)") 371 372 # touch status file for nagios 373 dirs = self.query_abs_dirs() 374 status_path = os.path.join(dirs["base_work_dir"], self.config["status_path"]) 375 self._touch_file(status_path) 376 377 378# __main__ {{{1 379if __name__ == "__main__": 380 bumper = L10nBumper() 381 bumper.run_and_exit() 382