1#!/usr/bin/env python 2# ***** BEGIN LICENSE BLOCK ***** 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this file, 5# You can obtain one at http://mozilla.org/MPL/2.0/. 6# ***** END LICENSE BLOCK ***** 7"""desktop_l10n.py 8 9This script manages Desktop repacks for nightly builds. 10""" 11import os 12import glob 13import re 14import sys 15import shlex 16 17# load modules from parent dir 18sys.path.insert(1, os.path.dirname(sys.path[0])) # noqa 19 20from mozharness.base.errors import MakefileErrorList 21from mozharness.base.script import BaseScript 22from mozharness.base.vcs.vcsbase import VCSMixin 23from mozharness.mozilla.automation import AutomationMixin 24from mozharness.mozilla.building.buildbase import ( 25 MakeUploadOutputParser, 26 get_mozconfig_path, 27) 28from mozharness.mozilla.l10n.locales import LocalesMixin 29 30try: 31 import simplejson as json 32 assert json 33except ImportError: 34 import json 35 36 37# needed by _map 38SUCCESS = 0 39FAILURE = 1 40 41SUCCESS_STR = "Success" 42FAILURE_STR = "Failed" 43 44 45# some other values such as "%(version)s", ... 46# are defined at run time and they cannot be enforced in the _pre_config_lock 47# phase 48runtime_config_tokens = ('version', 'locale', 'abs_objdir', 49 'en_us_installer_binary_url') 50 51 52# DesktopSingleLocale {{{1 53class DesktopSingleLocale(LocalesMixin, AutomationMixin, 54 VCSMixin, BaseScript): 55 """Manages desktop repacks""" 56 config_options = [[ 57 ['--locale', ], 58 {"action": "extend", 59 "dest": "locales", 60 "type": "string", 61 "help": "Specify the locale(s) to sign and update. Optionally pass" 62 " revision separated by colon, en-GB:default."} 63 ], [ 64 ['--tag-override', ], 65 {"action": "store", 66 "dest": "tag_override", 67 "type": "string", 68 "help": "Override the tags set for all repos"} 69 ], [ 70 ['--en-us-installer-url', ], 71 {"action": "store", 72 "dest": "en_us_installer_url", 73 "type": "string", 74 "help": "Specify the url of the en-us binary"} 75 ], [ 76 ['--scm-level'], { # Ignored on desktop for now: see Bug 1414678. 77 "action": "store", 78 "type": "int", 79 "dest": "scm_level", 80 "default": 1, 81 "help": "This sets the SCM level for the branch being built." 82 " See https://www.mozilla.org/en-US/about/" 83 "governance/policies/commit/access-policy/"} 84 ]] 85 86 def __init__(self, require_config_file=True): 87 # fxbuild style: 88 buildscript_kwargs = { 89 'all_actions': [ 90 "clone-locales", 91 "list-locales", 92 "setup", 93 "repack", 94 "summary", 95 ], 96 'config': { 97 "ignore_locales": ["en-US"], 98 "locales_dir": "browser/locales", 99 "log_name": "single_locale", 100 "hg_l10n_base": "https://hg.mozilla.org/l10n-central", 101 }, 102 } 103 104 LocalesMixin.__init__(self) 105 BaseScript.__init__( 106 self, 107 config_options=self.config_options, 108 require_config_file=require_config_file, 109 **buildscript_kwargs 110 ) 111 112 self.bootstrap_env = None 113 self.upload_env = None 114 self.upload_urls = {} 115 self.pushdate = None 116 # upload_files is a dictionary of files to upload, keyed by locale. 117 self.upload_files = {} 118 119 def _pre_config_lock(self, rw_config): 120 """replaces 'configuration_tokens' with their values, before the 121 configuration gets locked. If some of the configuration_tokens 122 are not present, stops the execution of the script""" 123 # now, only runtime_config_tokens should be present in config 124 # we should parse self.config and fail if any other we spot any 125 # other token 126 tokens_left = set(self._get_configuration_tokens(self.config)) 127 unknown_tokens = set(tokens_left) - set(runtime_config_tokens) 128 if unknown_tokens: 129 msg = ['unknown tokens in configuration:'] 130 for t in unknown_tokens: 131 msg.append(t) 132 self.fatal(' '.join(msg)) 133 self.info('configuration looks ok') 134 return 135 136 def _get_configuration_tokens(self, iterable): 137 """gets a list of tokens in iterable""" 138 regex = re.compile('%\(\w+\)s') 139 results = [] 140 try: 141 for element in iterable: 142 if isinstance(iterable, str): 143 # this is a string, look for tokens 144 # self.debug("{0}".format(re.findall(regex, element))) 145 tokens = re.findall(regex, iterable) 146 for token in tokens: 147 # clean %(branch)s => branch 148 # remove %( 149 token_name = token.partition('%(')[2] 150 # remove )s 151 token_name = token_name.partition(')s')[0] 152 results.append(token_name) 153 break 154 155 elif isinstance(iterable, (list, tuple)): 156 results.extend(self._get_configuration_tokens(element)) 157 158 elif isinstance(iterable, dict): 159 results.extend(self._get_configuration_tokens(iterable[element])) 160 161 except TypeError: 162 # element is a int/float/..., nothing to do here 163 pass 164 165 # remove duplicates, and return results 166 167 return list(set(results)) 168 169 def __detokenise_element(self, config_option, token, value): 170 """reads config_options and returns a version of the same config_option 171 replacing token with value recursively""" 172 # config_option is a string, let's replace token with value 173 if isinstance(config_option, str): 174 # if token does not appear in this string, 175 # nothing happens and the original value is returned 176 return config_option.replace(token, value) 177 # it's a dictionary 178 elif isinstance(config_option, dict): 179 # replace token for each element of this dictionary 180 for element in config_option: 181 config_option[element] = self.__detokenise_element( 182 config_option[element], token, value) 183 return config_option 184 # it's a list 185 elif isinstance(config_option, list): 186 # create a new list and append the replaced elements 187 new_list = [] 188 for element in config_option: 189 new_list.append(self.__detokenise_element(element, token, value)) 190 return new_list 191 elif isinstance(config_option, tuple): 192 # create a new list and append the replaced elements 193 new_list = [] 194 for element in config_option: 195 new_list.append(self.__detokenise_element(element, token, value)) 196 return tuple(new_list) 197 else: 198 # everything else, bool, number, ... 199 return config_option 200 201 # Helper methods {{{2 202 def query_bootstrap_env(self): 203 """returns the env for repacks""" 204 if self.bootstrap_env: 205 return self.bootstrap_env 206 config = self.config 207 replace_dict = self.query_abs_dirs() 208 209 bootstrap_env = self.query_env(partial_env=config.get("bootstrap_env"), 210 replace_dict=replace_dict) 211 if self.query_is_nightly(): 212 bootstrap_env["IS_NIGHTLY"] = "yes" 213 # we might set update_channel explicitly 214 if config.get('update_channel'): 215 update_channel = config['update_channel'] 216 else: # Let's just give the generic channel based on branch. 217 update_channel = "nightly-%s" % (config['branch'],) 218 if isinstance(update_channel, unicode): 219 update_channel = update_channel.encode("utf-8") 220 bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel 221 self.info("Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"])) 222 self.bootstrap_env = bootstrap_env 223 return self.bootstrap_env 224 225 def _query_upload_env(self): 226 """returns the environment used for the upload step""" 227 if self.upload_env: 228 return self.upload_env 229 config = self.config 230 231 upload_env = self.query_env(partial_env=config.get("upload_env")) 232 # check if there are any extra option from the platform configuration 233 # and append them to the env 234 235 if 'upload_env_extra' in config: 236 for extra in config['upload_env_extra']: 237 upload_env[extra] = config['upload_env_extra'][extra] 238 239 self.upload_env = upload_env 240 return self.upload_env 241 242 def query_l10n_env(self): 243 l10n_env = self._query_upload_env().copy() 244 l10n_env.update(self.query_bootstrap_env()) 245 return l10n_env 246 247 def _query_make_variable(self, variable, make_args=None): 248 """returns the value of make echo-variable-<variable> 249 it accepts extra make arguements (make_args) 250 """ 251 dirs = self.query_abs_dirs() 252 make_args = make_args or [] 253 target = ["echo-variable-%s" % variable] + make_args 254 cwd = dirs['abs_locales_dir'] 255 raw_output = self._get_output_from_make(target, cwd=cwd, 256 env=self.query_bootstrap_env()) 257 # we want to log all the messages from make 258 output = [] 259 for line in raw_output.split("\n"): 260 output.append(line.strip()) 261 output = " ".join(output).strip() 262 self.info('echo-variable-%s: %s' % (variable, output)) 263 return output 264 265 def _map(self, func, items): 266 """runs func for any item in items, calls the add_failure() for each 267 error. It assumes that function returns 0 when successful. 268 returns a two element tuple with (success_count, total_count)""" 269 success_count = 0 270 total_count = len(items) 271 name = func.__name__ 272 for item in items: 273 result = func(item) 274 if result == SUCCESS: 275 # success! 276 success_count += 1 277 else: 278 # func failed... 279 message = 'failure: %s(%s)' % (name, item) 280 self.add_failure(item, message) 281 return (success_count, total_count) 282 283 # Actions {{{2 284 def clone_locales(self): 285 self.pull_locale_source() 286 287 def setup(self): 288 """setup step""" 289 self._run_tooltool() 290 self._copy_mozconfig() 291 self._mach_configure() 292 self._run_make_in_config_dir() 293 self.make_wget_en_US() 294 self.make_unpack_en_US() 295 296 def _run_make_in_config_dir(self): 297 """this step creates nsinstall, needed my make_wget_en_US() 298 """ 299 dirs = self.query_abs_dirs() 300 config_dir = os.path.join(dirs['abs_objdir'], 'config') 301 env = self.query_bootstrap_env() 302 return self._make(target=['export'], cwd=config_dir, env=env) 303 304 def _copy_mozconfig(self): 305 """copies the mozconfig file into abs_mozilla_dir/.mozconfig 306 and logs the content 307 """ 308 config = self.config 309 dirs = self.query_abs_dirs() 310 src = get_mozconfig_path(self, config, dirs) 311 dst = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig') 312 self.copyfile(src, dst) 313 self.read_from_file(dst, verbose=True) 314 315 def _mach(self, target, env, halt_on_failure=True, output_parser=None): 316 dirs = self.query_abs_dirs() 317 mach = self._get_mach_executable() 318 return self.run_command(mach + target, 319 halt_on_failure=True, 320 env=env, 321 cwd=dirs['abs_mozilla_dir'], 322 output_parser=None) 323 324 def _mach_configure(self): 325 """calls mach configure""" 326 env = self.query_bootstrap_env() 327 target = ["configure"] 328 return self._mach(target=target, env=env) 329 330 def _get_mach_executable(self): 331 return [sys.executable, 'mach'] 332 333 def _get_make_executable(self): 334 config = self.config 335 dirs = self.query_abs_dirs() 336 if config.get('enable_mozmake'): # e.g. windows 337 make = r"/".join([dirs['abs_mozilla_dir'], 'mozmake.exe']) 338 # mysterious subprocess errors, let's try to fix this path... 339 make = make.replace('\\', '/') 340 make = [make] 341 else: 342 make = ['make'] 343 return make 344 345 def _make(self, target, cwd, env, error_list=MakefileErrorList, 346 halt_on_failure=True, output_parser=None): 347 """Runs make. Returns the exit code""" 348 make = self._get_make_executable() 349 if target: 350 make = make + target 351 return self.run_command(make, 352 cwd=cwd, 353 env=env, 354 error_list=error_list, 355 halt_on_failure=halt_on_failure, 356 output_parser=output_parser) 357 358 def _get_output_from_make(self, target, cwd, env, halt_on_failure=True, ignore_errors=False): 359 """runs make and returns the output of the command""" 360 make = self._get_make_executable() 361 return self.get_output_from_command(make + target, 362 cwd=cwd, 363 env=env, 364 silent=True, 365 halt_on_failure=halt_on_failure, 366 ignore_errors=ignore_errors) 367 368 def make_unpack_en_US(self): 369 """wrapper for make unpack""" 370 config = self.config 371 dirs = self.query_abs_dirs() 372 env = self.query_bootstrap_env() 373 cwd = os.path.join(dirs['abs_objdir'], config['locales_dir']) 374 return self._make(target=["unpack"], cwd=cwd, env=env) 375 376 def make_wget_en_US(self): 377 """wrapper for make wget-en-US""" 378 env = self.query_bootstrap_env() 379 dirs = self.query_abs_dirs() 380 cwd = dirs['abs_locales_dir'] 381 return self._make(target=["wget-en-US"], cwd=cwd, env=env) 382 383 def make_upload(self, locale): 384 """wrapper for make upload command""" 385 env = self.query_l10n_env() 386 dirs = self.query_abs_dirs() 387 target = ['upload', 'AB_CD=%s' % (locale)] 388 cwd = dirs['abs_locales_dir'] 389 parser = MakeUploadOutputParser(config=self.config, 390 log_obj=self.log_obj) 391 retval = self._make(target=target, cwd=cwd, env=env, 392 halt_on_failure=False, output_parser=parser) 393 if retval == SUCCESS: 394 self.info('Upload successful (%s)' % locale) 395 ret = SUCCESS 396 else: 397 self.error('failed to upload %s' % locale) 398 ret = FAILURE 399 400 if ret == FAILURE: 401 # If we failed above, we shouldn't even attempt a SIMPLE_NAME move 402 # even if we are configured to do so 403 return ret 404 405 # XXX Move the files to a SIMPLE_NAME format until we can enable 406 # Simple names in the build system 407 if self.config.get("simple_name_move"): 408 # Assume an UPLOAD PATH 409 upload_target = self.config["upload_env"]["UPLOAD_PATH"] 410 target_path = os.path.join(upload_target, locale) 411 self.mkdir_p(target_path) 412 glob_name = "*.%s.*" % locale 413 matches = (glob.glob(os.path.join(upload_target, glob_name)) + 414 glob.glob(os.path.join(upload_target, 'update', glob_name)) + 415 glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)) + 416 glob.glob(os.path.join(upload_target, 'install', 'sea', glob_name)) + 417 glob.glob(os.path.join(upload_target, 'setup.exe')) + 418 glob.glob(os.path.join(upload_target, 'setup-stub.exe'))) 419 targets_exts = ["tar.bz2", "dmg", "langpack.xpi", 420 "checksums", "zip", 421 "installer.exe", "installer-stub.exe"] 422 targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts] 423 targets.extend([(f, f) for f in 'setup.exe', 'setup-stub.exe']) 424 for f in matches: 425 possible_targets = [ 426 (tail, target_file) 427 for (tail, target_file) in targets 428 if f.endswith(tail) 429 ] 430 if len(possible_targets) == 1: 431 _, target_file = possible_targets[0] 432 # Remove from list of available options for this locale 433 targets.remove(possible_targets[0]) 434 else: 435 # wasn't valid (or already matched) 436 raise RuntimeError("Unexpected matching file name encountered: %s" 437 % f) 438 self.move(os.path.join(f), 439 os.path.join(target_path, target_file)) 440 self.log("Converted uploads for %s to simple names" % locale) 441 return ret 442 443 def set_upload_files(self, locale): 444 # The tree doesn't have a good way of exporting the list of files 445 # created during locale generation, but we can grab them by echoing the 446 # UPLOAD_FILES variable for each locale. 447 env = self.query_l10n_env() 448 target = ['echo-variable-UPLOAD_FILES', 'echo-variable-CHECKSUM_FILES', 449 'AB_CD=%s' % locale] 450 dirs = self.query_abs_dirs() 451 cwd = dirs['abs_locales_dir'] 452 # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found 453 # we should ignore stderr as unfortunately it's expected when parsing for values 454 output = self._get_output_from_make(target=target, cwd=cwd, env=env, 455 ignore_errors=True) 456 self.info('UPLOAD_FILES is "%s"' % output) 457 files = shlex.split(output) 458 if not files: 459 self.error('failed to get upload file list for locale %s' % locale) 460 return FAILURE 461 462 self.upload_files[locale] = [ 463 os.path.abspath(os.path.join(cwd, f)) for f in files 464 ] 465 return SUCCESS 466 467 def make_installers(self, locale): 468 """wrapper for make installers-(locale)""" 469 env = self.query_l10n_env() 470 self._copy_mozconfig() 471 dirs = self.query_abs_dirs() 472 cwd = os.path.join(dirs['abs_locales_dir']) 473 target = ["installers-%s" % locale, ] 474 return self._make(target=target, cwd=cwd, 475 env=env, halt_on_failure=False) 476 477 def repack_locale(self, locale): 478 """wraps the logic for make installers and generating 479 complete updates.""" 480 481 # run make installers 482 if self.make_installers(locale) != SUCCESS: 483 self.error("make installers-%s failed" % (locale)) 484 return FAILURE 485 486 # now try to upload the artifacts 487 if self.make_upload(locale): 488 self.error("make upload for locale %s failed!" % (locale)) 489 return FAILURE 490 491 # set_upload_files() should be called after make upload, to make sure 492 # we have all files in place (checksums, etc) 493 if self.set_upload_files(locale): 494 self.error("failed to get list of files to upload for locale %s" % locale) 495 return FAILURE 496 497 return SUCCESS 498 499 def repack(self): 500 """creates the repacks and udpates""" 501 self._map(self.repack_locale, self.query_locales()) 502 503 def _query_objdir(self): 504 """returns objdir name from configuration""" 505 return self.config['objdir'] 506 507 def query_abs_dirs(self): 508 if self.abs_dirs: 509 return self.abs_dirs 510 abs_dirs = super(DesktopSingleLocale, self).query_abs_dirs() 511 for directory in abs_dirs: 512 value = abs_dirs[directory] 513 abs_dirs[directory] = value 514 dirs = {} 515 dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools') 516 dirs['abs_src_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src') 517 for key in dirs.keys(): 518 if key not in abs_dirs: 519 abs_dirs[key] = dirs[key] 520 self.abs_dirs = abs_dirs 521 return self.abs_dirs 522 523 # TODO: replace with ToolToolMixin 524 def _get_tooltool_auth_file(self): 525 # set the default authentication file based on platform; this 526 # corresponds to where puppet puts the token 527 if 'tooltool_authentication_file' in self.config: 528 fn = self.config['tooltool_authentication_file'] 529 elif self._is_windows(): 530 fn = r'c:\builds\relengapi.tok' 531 else: 532 fn = '/builds/relengapi.tok' 533 534 # if the file doesn't exist, don't pass it to tooltool (it will just 535 # fail). In taskcluster, this will work OK as the relengapi-proxy will 536 # take care of auth. Everywhere else, we'll get auth failures if 537 # necessary. 538 if os.path.exists(fn): 539 return fn 540 541 def _run_tooltool(self): 542 env = self.query_bootstrap_env() 543 config = self.config 544 dirs = self.query_abs_dirs() 545 toolchains = os.environ.get('MOZ_TOOLCHAINS') 546 manifest_src = os.environ.get('TOOLTOOL_MANIFEST') 547 if not manifest_src: 548 manifest_src = config.get('tooltool_manifest_src') 549 if not manifest_src and not toolchains: 550 return 551 python = sys.executable 552 553 cmd = [ 554 python, '-u', 555 os.path.join(dirs['abs_mozilla_dir'], 'mach'), 556 'artifact', 557 'toolchain', 558 '-v', 559 '--retry', '4', 560 '--artifact-manifest', 561 os.path.join(dirs['abs_mozilla_dir'], 'toolchains.json'), 562 ] 563 if manifest_src: 564 cmd.extend([ 565 '--tooltool-manifest', 566 os.path.join(dirs['abs_mozilla_dir'], manifest_src), 567 '--tooltool-url', 568 config['tooltool_url'], 569 ]) 570 auth_file = self._get_tooltool_auth_file() 571 if auth_file and os.path.exists(auth_file): 572 cmd.extend(['--authentication-file', auth_file]) 573 cache = config['bootstrap_env'].get('TOOLTOOL_CACHE') 574 if cache: 575 cmd.extend(['--cache-dir', cache]) 576 if toolchains: 577 cmd.extend(toolchains.split()) 578 self.info(str(cmd)) 579 self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True, 580 env=env) 581 582 583# main {{{ 584if __name__ == '__main__': 585 single_locale = DesktopSingleLocale() 586 single_locale.run_and_exit() 587