1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this, 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7from appdirs import user_config_dir 8from hglib.error import CommandError 9from mach.decorators import ( 10 CommandArgument, 11 CommandProvider, 12 Command, 13) 14from mach.base import FailedCommandError 15from mozbuild.base import MachCommandBase 16from mozrelease.scriptworker_canary import get_secret 17from pathlib import Path 18from redo import retry 19import argparse 20import logging 21import os 22import tempfile 23 24 25@CommandProvider 26class CompareLocales(MachCommandBase): 27 """Run compare-locales.""" 28 29 @Command( 30 "compare-locales", 31 category="build", 32 description="Run source checks on a localization.", 33 ) 34 @CommandArgument( 35 "config_paths", 36 metavar="l10n.toml", 37 nargs="+", 38 help="TOML or INI file for the project", 39 ) 40 @CommandArgument( 41 "l10n_base_dir", 42 metavar="l10n-base-dir", 43 help="Parent directory of localizations", 44 ) 45 @CommandArgument( 46 "locales", 47 nargs="*", 48 metavar="locale-code", 49 help="Locale code and top-level directory of " "each localization", 50 ) 51 @CommandArgument( 52 "-q", 53 "--quiet", 54 action="count", 55 default=0, 56 help="""Show less data. 57Specified once, don't show obsolete entities. Specified twice, also hide 58missing entities. Specify thrice to exclude warnings and four times to 59just show stats""", 60 ) 61 @CommandArgument( 62 "-m", "--merge", help="""Use this directory to stage merged files""" 63 ) 64 @CommandArgument( 65 "--validate", action="store_true", help="Run compare-locales against reference" 66 ) 67 @CommandArgument( 68 "--json", 69 help="""Serialize to JSON. Value is the name of 70the output file, pass "-" to serialize to stdout and hide the default output. 71""", 72 ) 73 @CommandArgument( 74 "-D", 75 action="append", 76 metavar="var=value", 77 default=[], 78 dest="defines", 79 help="Overwrite variables in TOML files", 80 ) 81 @CommandArgument( 82 "--full", action="store_true", help="Compare projects that are disabled" 83 ) 84 @CommandArgument( 85 "--return-zero", action="store_true", help="Return 0 regardless of l10n status" 86 ) 87 def compare(self, command_context, **kwargs): 88 from compare_locales.commands import CompareLocales 89 90 class ErrorHelper(object): 91 """Dummy ArgumentParser to marshall compare-locales 92 commandline errors to mach exceptions. 93 """ 94 95 def error(self, msg): 96 raise FailedCommandError(msg) 97 98 def exit(self, message=None, status=0): 99 raise FailedCommandError(message, exit_code=status) 100 101 cmd = CompareLocales() 102 cmd.parser = ErrorHelper() 103 return cmd.handle(**kwargs) 104 105 106# https://stackoverflow.com/a/14117511 107def _positive_int(value): 108 value = int(value) 109 if value <= 0: 110 raise argparse.ArgumentTypeError(f"{value} must be a positive integer.") 111 return value 112 113 114class RetryError(Exception): 115 ... 116 117 118VCT_PATH = Path(".").resolve() / "vct" 119VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/" 120FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree" 121HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc") 122 123 124@CommandProvider 125class CrossChannel(MachCommandBase): 126 """Run l10n cross-channel content generation.""" 127 128 @Command( 129 "l10n-cross-channel", 130 category="misc", 131 description="Create cross-channel content.", 132 ) 133 @CommandArgument( 134 "--strings-path", 135 "-s", 136 metavar="en-US", 137 type=Path, 138 default=Path("en-US"), 139 help="Path to mercurial repository for gecko-strings-quarantine", 140 ) 141 @CommandArgument( 142 "--outgoing-path", 143 "-o", 144 type=Path, 145 help="create an outgoing() patch if there are changes", 146 ) 147 @CommandArgument( 148 "--attempts", 149 type=_positive_int, 150 default=1, 151 help="Number of times to try (for automation)", 152 ) 153 @CommandArgument( 154 "--ssh-secret", 155 action="store", 156 help="Taskcluster secret to use to push (for automation)", 157 ) 158 @CommandArgument( 159 "actions", 160 choices=("prep", "create", "push"), 161 nargs="+", 162 # This help block will be poorly formatted until we fix bug 1714239 163 help=""" 164 "prep": clone repos and pull heads. 165 "create": create the en-US strings commit an optionally create an 166 outgoing() patch. 167 "push": push the en-US strings to the quarantine repo. 168 """, 169 ) 170 def cross_channel( 171 self, 172 command_context, 173 strings_path, 174 outgoing_path, 175 actions, 176 attempts, 177 ssh_secret, 178 **kwargs, 179 ): 180 # This can be any path, as long as the name of the directory is en-US. 181 # Not entirely sure where this is a requirement; perhaps in l10n 182 # string manipulation logic? 183 if strings_path.name != "en-US": 184 raise FailedCommandError("strings_path needs to be named `en-US`") 185 self.activate_virtualenv() 186 # XXX pin python requirements 187 self.virtualenv_manager.install_pip_requirements( 188 Path(os.path.dirname(__file__)) / "requirements.in" 189 ) 190 strings_path = strings_path.resolve() # abspath 191 if outgoing_path: 192 outgoing_path = outgoing_path.resolve() # abspath 193 try: 194 with tempfile.TemporaryDirectory() as ssh_key_dir: 195 retry( 196 self._do_create_content, 197 attempts=attempts, 198 retry_exceptions=(RetryError,), 199 args=( 200 command_context, 201 strings_path, 202 outgoing_path, 203 ssh_secret, 204 Path(ssh_key_dir), 205 actions, 206 ), 207 ) 208 except RetryError as exc: 209 raise FailedCommandError(exc) from exc 210 211 def _do_create_content( 212 self, 213 command_context, 214 strings_path, 215 outgoing_path, 216 ssh_secret, 217 ssh_key_dir, 218 actions, 219 ): 220 221 from mozxchannel import CrossChannelCreator, get_default_config 222 223 config = get_default_config(Path(self.topsrcdir), strings_path) 224 ccc = CrossChannelCreator(config) 225 status = 0 226 changes = False 227 ssh_key_secret = None 228 ssh_key_file = None 229 230 if "prep" in actions: 231 if ssh_secret: 232 if not os.environ.get("MOZ_AUTOMATION"): 233 raise CommandError( 234 "I don't know how to fetch the ssh secret outside of automation!" 235 ) 236 ssh_key_secret = get_secret(ssh_secret) 237 ssh_key_file = ssh_key_dir.joinpath("id_rsa") 238 ssh_key_file.write_text(ssh_key_secret["ssh_privkey"]) 239 ssh_key_file.chmod(0o600) 240 # Set up firefoxtree for comm per bug 1659691 comment 22 241 if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists(): 242 self._clone_hg_repo(VCT_URL, VCT_PATH) 243 hgrc_content = [ 244 "[extensions]", 245 f"firefoxtree = {FXTREE_PATH}", 246 "", 247 "[ui]", 248 "username = trybld", 249 ] 250 if ssh_key_file: 251 hgrc_content.extend( 252 [ 253 f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}", 254 ] 255 ) 256 HGRC_PATH.write_text("\n".join(hgrc_content)) 257 if strings_path.exists() and self._check_outgoing(strings_path): 258 self._strip_outgoing(strings_path) 259 # Clone strings + source repos, pull heads 260 for repo_config in (config["strings"], *config["source"].values()): 261 if not repo_config["path"].exists(): 262 self._clone_hg_repo(repo_config["url"], str(repo_config["path"])) 263 for head in repo_config["heads"].keys(): 264 command = ["hg", "--cwd", str(repo_config["path"]), "pull"] 265 command.append(head) 266 status = self._retry_run_process(command, ensure_exit_code=False) 267 if status not in (0, 255): # 255 on pull with no changes 268 raise RetryError(f"Failure on pull: status {status}!") 269 if repo_config.get("update_on_pull"): 270 command = [ 271 "hg", 272 "--cwd", 273 str(repo_config["path"]), 274 "up", 275 "-C", 276 "-r", 277 head, 278 ] 279 status = self._retry_run_process( 280 command, ensure_exit_code=False 281 ) 282 if status not in (0, 255): # 255 on pull with no changes 283 raise RetryError(f"Failure on update: status {status}!") 284 self._check_hg_repo( 285 repo_config["path"], heads=repo_config.get("heads", {}).keys() 286 ) 287 else: 288 self._check_hg_repo(strings_path) 289 for repo_config in config.get("source", {}).values(): 290 self._check_hg_repo( 291 repo_config["path"], heads=repo_config.get("heads", {}).keys() 292 ) 293 if self._check_outgoing(strings_path): 294 raise RetryError(f"check: Outgoing changes in {strings_path}!") 295 296 if "create" in actions: 297 try: 298 status = ccc.create_content() 299 changes = True 300 self._create_outgoing_patch(outgoing_path, strings_path) 301 except CommandError as exc: 302 if exc.ret != 1: 303 raise RetryError(exc) from exc 304 self.log(logging.INFO, "create", {}, "No new strings.") 305 306 if "push" in actions: 307 if changes: 308 self._retry_run_process( 309 [ 310 "hg", 311 "--cwd", 312 str(strings_path), 313 "push", 314 "-r", 315 ".", 316 config["strings"]["push_url"], 317 ], 318 line_handler=print, 319 ) 320 else: 321 self.log(logging.INFO, "push", {}, "Skipping empty push.") 322 323 return status 324 325 def _check_outgoing(self, strings_path): 326 status = self._retry_run_process( 327 ["hg", "--cwd", str(strings_path), "out", "-r", "."], 328 ensure_exit_code=False, 329 ) 330 if status == 0: 331 return True 332 if status == 1: 333 return False 334 raise RetryError( 335 f"Outgoing check in {strings_path} returned unexpected {status}!" 336 ) 337 338 def _strip_outgoing(self, strings_path): 339 self._retry_run_process( 340 [ 341 "hg", 342 "--config", 343 "extensions.strip=", 344 "--cwd", 345 str(strings_path), 346 "strip", 347 "--no-backup", 348 "outgoing()", 349 ], 350 ) 351 352 def _create_outgoing_patch(self, path, strings_path): 353 if not path: 354 return 355 if not path.parent.exists(): 356 os.makedirs(path.parent) 357 with open(path, "w") as fh: 358 359 def writeln(line): 360 fh.write(f"{line}\n") 361 362 self._retry_run_process( 363 [ 364 "hg", 365 "--cwd", 366 str(strings_path), 367 "log", 368 "--patch", 369 "--verbose", 370 "-r", 371 "outgoing()", 372 ], 373 line_handler=writeln, 374 ) 375 376 def _retry_run_process(self, *args, error_msg=None, **kwargs): 377 try: 378 return self.run_process(*args, **kwargs) 379 except Exception as exc: 380 raise RetryError(error_msg or str(exc)) from exc 381 382 def _check_hg_repo(self, path, heads=None): 383 if not (path.is_dir() and (path / ".hg").is_dir()): 384 raise RetryError(f"{path} is not a Mercurial repository") 385 if heads: 386 for head in heads: 387 self._retry_run_process( 388 ["hg", "--cwd", str(path), "log", "-r", head], 389 error_msg=f"check: {path} has no head {head}!", 390 ) 391 392 def _clone_hg_repo(self, url, path): 393 self._retry_run_process(["hg", "clone", url, str(path)]) 394