1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7"""desktop_l10n.py 8 9This script manages Desktop repacks for nightly builds. 10""" 11from __future__ import absolute_import 12import os 13import glob 14import sys 15import shlex 16 17# load modules from parent dir 18sys.path.insert(1, os.path.dirname(sys.path[0])) # noqa 19 20from mozharness.base.errors import MakefileErrorList 21from mozharness.base.script import BaseScript 22from mozharness.base.vcs.vcsbase import VCSMixin 23from mozharness.mozilla.automation import AutomationMixin 24from mozharness.mozilla.building.buildbase import ( 25 MakeUploadOutputParser, 26 get_mozconfig_path, 27) 28from mozharness.mozilla.l10n.locales import LocalesMixin 29 30try: 31 import simplejson as json 32 33 assert json 34except ImportError: 35 import json 36 37 38# needed by _map 39SUCCESS = 0 40FAILURE = 1 41 42SUCCESS_STR = "Success" 43FAILURE_STR = "Failed" 44 45 46# DesktopSingleLocale {{{1 47class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): 48 """Manages desktop repacks""" 49 50 config_options = [ 51 [ 52 [ 53 "--locale", 54 ], 55 { 56 "action": "extend", 57 "dest": "locales", 58 "type": "string", 59 "help": "Specify the locale(s) to sign and update. Optionally pass" 60 " revision separated by colon, en-GB:default.", 61 }, 62 ], 63 [ 64 [ 65 "--tag-override", 66 ], 67 { 68 "action": "store", 69 "dest": "tag_override", 70 "type": "string", 71 "help": "Override the tags set for all repos", 72 }, 73 ], 74 [ 75 [ 76 "--en-us-installer-url", 77 ], 78 { 79 "action": "store", 80 "dest": "en_us_installer_url", 81 "type": "string", 82 "help": "Specify the url of the en-us binary", 83 }, 84 ], 85 ] 86 87 def __init__(self, require_config_file=True): 88 # fxbuild style: 89 buildscript_kwargs = { 90 "all_actions": [ 91 "clone-locales", 92 "list-locales", 93 "setup", 94 "repack", 95 "summary", 96 ], 97 "config": { 98 "ignore_locales": ["en-US"], 99 "locales_dir": "browser/locales", 100 "log_name": "single_locale", 101 "hg_l10n_base": "https://hg.mozilla.org/l10n-central", 102 }, 103 } 104 105 LocalesMixin.__init__(self) 106 BaseScript.__init__( 107 self, 108 config_options=self.config_options, 109 require_config_file=require_config_file, 110 **buildscript_kwargs 111 ) 112 113 self.bootstrap_env = None 114 self.upload_env = None 115 self.upload_urls = {} 116 self.pushdate = None 117 # upload_files is a dictionary of files to upload, keyed by locale. 118 self.upload_files = {} 119 120 # Helper methods {{{2 121 def query_bootstrap_env(self): 122 """returns the env for repacks""" 123 if self.bootstrap_env: 124 return self.bootstrap_env 125 config = self.config 126 abs_dirs = self.query_abs_dirs() 127 128 bootstrap_env = self.query_env( 129 partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs 130 ) 131 132 bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"] 133 if self.query_is_nightly(): 134 # we might set update_channel explicitly 135 if config.get("update_channel"): 136 update_channel = config["update_channel"] 137 else: # Let's just give the generic channel based on branch. 138 update_channel = "nightly-%s" % (config["branch"],) 139 if not isinstance(update_channel, bytes): 140 update_channel = update_channel.encode("utf-8") 141 bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel 142 self.info( 143 "Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"]) 144 ) 145 self.bootstrap_env = bootstrap_env 146 return self.bootstrap_env 147 148 def _query_upload_env(self): 149 """returns the environment used for the upload step""" 150 if self.upload_env: 151 return self.upload_env 152 config = self.config 153 154 upload_env = self.query_env(partial_env=config.get("upload_env")) 155 # check if there are any extra option from the platform configuration 156 # and append them to the env 157 158 if "upload_env_extra" in config: 159 for extra in config["upload_env_extra"]: 160 upload_env[extra] = config["upload_env_extra"][extra] 161 162 self.upload_env = upload_env 163 return self.upload_env 164 165 def query_l10n_env(self): 166 l10n_env = self._query_upload_env().copy() 167 l10n_env.update(self.query_bootstrap_env()) 168 return l10n_env 169 170 def _query_make_variable(self, variable, make_args=None): 171 """returns the value of make echo-variable-<variable> 172 it accepts extra make arguements (make_args) 173 """ 174 dirs = self.query_abs_dirs() 175 make_args = make_args or [] 176 target = ["echo-variable-%s" % variable] + make_args 177 cwd = dirs["abs_locales_dir"] 178 raw_output = self._get_output_from_make( 179 target, cwd=cwd, env=self.query_bootstrap_env() 180 ) 181 # we want to log all the messages from make 182 output = [] 183 for line in raw_output.split("\n"): 184 output.append(line.strip()) 185 output = " ".join(output).strip() 186 self.info("echo-variable-%s: %s" % (variable, output)) 187 return output 188 189 def _map(self, func, items): 190 """runs func for any item in items, calls the add_failure() for each 191 error. It assumes that function returns 0 when successful. 192 returns a two element tuple with (success_count, total_count)""" 193 success_count = 0 194 total_count = len(items) 195 name = func.__name__ 196 for item in items: 197 result = func(item) 198 if result == SUCCESS: 199 # success! 200 success_count += 1 201 else: 202 # func failed... 203 message = "failure: %s(%s)" % (name, item) 204 self.add_failure(item, message) 205 return (success_count, total_count) 206 207 # Actions {{{2 208 def clone_locales(self): 209 self.pull_locale_source() 210 211 def setup(self): 212 """setup step""" 213 self._run_tooltool() 214 self._copy_mozconfig() 215 self._mach_configure() 216 self._run_make_in_config_dir() 217 self.make_wget_en_US() 218 self.make_unpack_en_US() 219 220 def _run_make_in_config_dir(self): 221 """this step creates nsinstall, needed my make_wget_en_US()""" 222 dirs = self.query_abs_dirs() 223 config_dir = os.path.join(dirs["abs_obj_dir"], "config") 224 env = self.query_bootstrap_env() 225 return self._make(target=["export"], cwd=config_dir, env=env) 226 227 def _copy_mozconfig(self): 228 """copies the mozconfig file into abs_src_dir/.mozconfig 229 and logs the content 230 """ 231 config = self.config 232 dirs = self.query_abs_dirs() 233 src = get_mozconfig_path(self, config, dirs) 234 dst = os.path.join(dirs["abs_src_dir"], ".mozconfig") 235 self.copyfile(src, dst) 236 self.read_from_file(dst, verbose=True) 237 238 def _mach(self, target, env, halt_on_failure=True, output_parser=None): 239 dirs = self.query_abs_dirs() 240 mach = self._get_mach_executable() 241 return self.run_command( 242 mach + target, 243 halt_on_failure=True, 244 env=env, 245 cwd=dirs["abs_src_dir"], 246 output_parser=None, 247 ) 248 249 def _mach_configure(self): 250 """calls mach configure""" 251 env = self.query_bootstrap_env() 252 target = ["configure"] 253 return self._mach(target=target, env=env) 254 255 def _get_mach_executable(self): 256 return [sys.executable, "mach"] 257 258 def _make( 259 self, 260 target, 261 cwd, 262 env, 263 error_list=MakefileErrorList, 264 halt_on_failure=True, 265 output_parser=None, 266 ): 267 """Runs make. Returns the exit code""" 268 make = ["make"] 269 if target: 270 make = make + target 271 return self.run_command( 272 make, 273 cwd=cwd, 274 env=env, 275 error_list=error_list, 276 halt_on_failure=halt_on_failure, 277 output_parser=output_parser, 278 ) 279 280 def _get_output_from_make( 281 self, target, cwd, env, halt_on_failure=True, ignore_errors=False 282 ): 283 """runs make and returns the output of the command""" 284 return self.get_output_from_command( 285 ["make"] + target, 286 cwd=cwd, 287 env=env, 288 silent=True, 289 halt_on_failure=halt_on_failure, 290 ignore_errors=ignore_errors, 291 ) 292 293 def make_unpack_en_US(self): 294 """wrapper for make unpack""" 295 config = self.config 296 dirs = self.query_abs_dirs() 297 env = self.query_bootstrap_env() 298 cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"]) 299 return self._make(target=["unpack"], cwd=cwd, env=env) 300 301 def make_wget_en_US(self): 302 """wrapper for make wget-en-US""" 303 env = self.query_bootstrap_env() 304 dirs = self.query_abs_dirs() 305 cwd = dirs["abs_locales_dir"] 306 return self._make(target=["wget-en-US"], cwd=cwd, env=env) 307 308 def make_upload(self, locale): 309 """wrapper for make upload command""" 310 env = self.query_l10n_env() 311 dirs = self.query_abs_dirs() 312 target = ["upload", "AB_CD=%s" % (locale)] 313 cwd = dirs["abs_locales_dir"] 314 parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj) 315 retval = self._make( 316 target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser 317 ) 318 if retval == SUCCESS: 319 self.info("Upload successful (%s)" % locale) 320 ret = SUCCESS 321 else: 322 self.error("failed to upload %s" % locale) 323 ret = FAILURE 324 325 if ret == FAILURE: 326 # If we failed above, we shouldn't even attempt a SIMPLE_NAME move 327 # even if we are configured to do so 328 return ret 329 330 # XXX Move the files to a SIMPLE_NAME format until we can enable 331 # Simple names in the build system 332 if self.config.get("simple_name_move"): 333 # Assume an UPLOAD PATH 334 upload_target = self.config["upload_env"]["UPLOAD_PATH"] 335 target_path = os.path.join(upload_target, locale) 336 self.mkdir_p(target_path) 337 glob_name = "*.%s.*" % locale 338 matches = ( 339 glob.glob(os.path.join(upload_target, glob_name)) 340 + glob.glob(os.path.join(upload_target, "update", glob_name)) 341 + glob.glob(os.path.join(upload_target, "*", "xpi", glob_name)) 342 + glob.glob(os.path.join(upload_target, "install", "sea", glob_name)) 343 + glob.glob(os.path.join(upload_target, "setup.exe")) 344 + glob.glob(os.path.join(upload_target, "setup-stub.exe")) 345 ) 346 targets_exts = [ 347 "tar.bz2", 348 "dmg", 349 "langpack.xpi", 350 "checksums", 351 "zip", 352 "installer.exe", 353 "installer-stub.exe", 354 ] 355 targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts] 356 targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")]) 357 for f in matches: 358 possible_targets = [ 359 (tail, target_file) 360 for (tail, target_file) in targets 361 if f.endswith(tail) 362 ] 363 if len(possible_targets) == 1: 364 _, target_file = possible_targets[0] 365 # Remove from list of available options for this locale 366 targets.remove(possible_targets[0]) 367 else: 368 # wasn't valid (or already matched) 369 raise RuntimeError( 370 "Unexpected matching file name encountered: %s" % f 371 ) 372 self.move(os.path.join(f), os.path.join(target_path, target_file)) 373 self.log("Converted uploads for %s to simple names" % locale) 374 return ret 375 376 def set_upload_files(self, locale): 377 # The tree doesn't have a good way of exporting the list of files 378 # created during locale generation, but we can grab them by echoing the 379 # UPLOAD_FILES variable for each locale. 380 env = self.query_l10n_env() 381 target = [ 382 "echo-variable-UPLOAD_FILES", 383 "echo-variable-CHECKSUM_FILES", 384 "AB_CD=%s" % locale, 385 ] 386 dirs = self.query_abs_dirs() 387 cwd = dirs["abs_locales_dir"] 388 # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found 389 # we should ignore stderr as unfortunately it's expected when parsing for values 390 output = self._get_output_from_make( 391 target=target, cwd=cwd, env=env, ignore_errors=True 392 ) 393 self.info('UPLOAD_FILES is "%s"' % output) 394 files = shlex.split(output) 395 if not files: 396 self.error("failed to get upload file list for locale %s" % locale) 397 return FAILURE 398 399 self.upload_files[locale] = [ 400 os.path.abspath(os.path.join(cwd, f)) for f in files 401 ] 402 return SUCCESS 403 404 def make_installers(self, locale): 405 """wrapper for make installers-(locale)""" 406 env = self.query_l10n_env() 407 env["PYTHONIOENCODING"] = "utf-8" 408 self._copy_mozconfig() 409 dirs = self.query_abs_dirs() 410 cwd = os.path.join(dirs["abs_locales_dir"]) 411 target = [ 412 "installers-%s" % locale, 413 ] 414 return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False) 415 416 def repack_locale(self, locale): 417 """wraps the logic for make installers and generating 418 complete updates.""" 419 420 # run make installers 421 if self.make_installers(locale) != SUCCESS: 422 self.error("make installers-%s failed" % (locale)) 423 return FAILURE 424 425 # now try to upload the artifacts 426 if self.make_upload(locale): 427 self.error("make upload for locale %s failed!" % (locale)) 428 return FAILURE 429 430 # set_upload_files() should be called after make upload, to make sure 431 # we have all files in place (checksums, etc) 432 if self.set_upload_files(locale): 433 self.error("failed to get list of files to upload for locale %s" % locale) 434 return FAILURE 435 436 return SUCCESS 437 438 def repack(self): 439 """creates the repacks and udpates""" 440 self._map(self.repack_locale, self.query_locales()) 441 442 def _run_tooltool(self): 443 env = self.query_bootstrap_env() 444 config = self.config 445 dirs = self.query_abs_dirs() 446 manifest_src = os.environ.get("TOOLTOOL_MANIFEST") 447 if not manifest_src: 448 manifest_src = config.get("tooltool_manifest_src") 449 if not manifest_src: 450 return 451 python = sys.executable 452 453 cmd = [ 454 python, 455 "-u", 456 os.path.join(dirs["abs_src_dir"], "mach"), 457 "artifact", 458 "toolchain", 459 "-v", 460 "--retry", 461 "4", 462 "--artifact-manifest", 463 os.path.join(dirs["abs_src_dir"], "toolchains.json"), 464 ] 465 if manifest_src: 466 cmd.extend( 467 [ 468 "--tooltool-manifest", 469 os.path.join(dirs["abs_src_dir"], manifest_src), 470 ] 471 ) 472 cache = config["bootstrap_env"].get("TOOLTOOL_CACHE") 473 if cache: 474 cmd.extend(["--cache-dir", cache]) 475 self.info(str(cmd)) 476 self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env) 477 478 479# main {{{ 480if __name__ == "__main__": 481 single_locale = DesktopSingleLocale() 482 single_locale.run_and_exit() 483